A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Hardware][Intel GPU] Add Intel GPU(XPU) inference backend (#3814)
Co-authored-by: Jiang Li <jiang1.li@intel.com> Co-authored-by: Abhilash Majumder <abhilash.majumder@intel.com> Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
Kunshang Ji committed
Jun 18, 2024 at 02:01 UTC
728c4c8a063c25e7a20d6eda20a3f30873bda4c6
31 files changed
+1998
-24
.buildkite/run-xpu-test.sh
new
+14
@@ -0,0 +1,14 @@
1
+# This script build the CPU docker image and run the offline inference inside the container.
2
+# It serves a sanity check for compilation and basic model usage.
3
+set -ex
4
+
5
+# Try building the docker image
6
+docker build -t xpu-test -f Dockerfile.xpu .
7
+
8
+# Setup cleanup
9
+remove_docker_container() { docker rm -f xpu-test || true; }
10
+trap remove_docker_container EXIT
11
+remove_docker_container
12
+
13
+# Run the image and launch offline inference
14
+docker run --network host --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path xpu-test python3 examples/offline_inference.py
.buildkite/test-template.j2
+5
@@ -45,6 +45,11 @@ steps:
45
queue: intel
46
command: bash .buildkite/run-cpu-test.sh
47
48
+ - label: "XPU Test"
49
+ agents:
50
+ queue: intel
51
+ command: bash .buildkite/run-xpu-test.sh
52
+
53
{% for step in steps %}
54
- label: "{{ step.label }}"
55
agents:
Dockerfile.xpu
new
+22
@@ -0,0 +1,22 @@
1
+FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04
2
+
3
+RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/intel-oneapi-archive-keyring.gpg > /dev/null && \
4
+ echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " | tee /etc/apt/sources.list.d/oneAPI.list && \
5
+ chmod 644 /usr/share/keyrings/intel-oneapi-archive-keyring.gpg && \
6
+ rm /etc/apt/sources.list.d/intel-graphics.list && \
7
+ wget -O- https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor | tee /usr/share/keyrings/intel-graphics.gpg > /dev/null && \
8
+ echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc" | tee /etc/apt/sources.list.d/intel.gpu.jammy.list && \
9
+ chmod 644 /usr/share/keyrings/intel-graphics.gpg
10
+
11
+RUN apt-get update -y \
12
+&& apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip
13
+
14
+COPY ./ /workspace/vllm
15
+
16
+WORKDIR /workspace/vllm
17
+
18
+RUN pip install -v -r requirements-xpu.txt
19
+
20
+RUN VLLM_TARGET_DEVICE=xpu python3 setup.py install
21
+
22
+CMD ["/bin/bash"]
benchmarks/benchmark_latency.py
+1
-1
@@ -191,7 +191,7 @@ if __name__ == '__main__':
191
"--device",
192
type=str,
193
default="cuda",
194
- choices=["cuda", "cpu", "tpu"],
194
+ choices=["cuda", "cpu", "tpu", "xpu"],
195
help='device type for vLLM execution, supporting CUDA and CPU.')
196
parser.add_argument('--block-size',
197
type=int,
benchmarks/benchmark_throughput.py
+1
-1
@@ -349,7 +349,7 @@ if __name__ == "__main__":
349
"--device",
350
type=str,
351
default="cuda",
352
- choices=["cuda", "cpu", "tpu"],
352
+ choices=["cuda", "cpu", "tpu", "xpu"],
353
help='device type for vLLM execution, supporting CUDA and CPU.')
354
parser.add_argument(
355
"--enable-prefix-caching",
docs/source/getting_started/xpu-installation.rst
new
+61
@@ -0,0 +1,61 @@
1
+.. _installation_xpu:
2
+
3
+Installation with XPU
4
+========================
5
+
6
+vLLM initially supports basic model inferencing and serving on Intel GPU platform.
7
+
8
+Table of contents:
9
+
10
+#. :ref:`Requirements <xpu_backend_requirements>`
11
+#. :ref:`Quick start using Dockerfile <xpu_backend_quick_start_dockerfile>`
12
+#. :ref:`Build from source <build_xpu_backend_from_source>`
13
+
14
+.. _xpu_backend_requirements:
15
+
16
+Requirements
17
+------------
18
+
19
+* OS: Linux
20
+* Supported Hardware: Intel Data Center GPU (Intel ARC GPU WIP)
21
+* OneAPI requirements: oneAPI 2024.1
22
+
23
+.. _xpu_backend_quick_start_dockerfile:
24
+
25
+Quick start using Dockerfile
26
+----------------------------
27
+
28
+.. code-block:: console
29
+
30
+ $ docker build -f Dockerfile.xpu -t vllm-xpu-env --shm-size=4g .
31
+ $ docker run -it \
32
+ --rm \
33
+ --network=host \
34
+ --device /dev/dri \
35
+ -v /dev/dri/by-path:/dev/dri/by-path \
36
+ vllm-xpu-env
37
+
38
+.. _build_xpu_backend_from_source:
39
+
40
+Build from source
41
+-----------------
42
+
43
+- First, install required driver and intel OneAPI 2024.1.
44
+
45
+- Second, install Python packages for vLLM XPU backend building:
46
+
47
+.. code-block:: console
48
+
49
+ $ pip install --upgrade pip
50
+ $ pip install -v -r requirements-xpu.txt
51
+
52
+- Finally, build and install vLLM XPU backend:
53
+
54
+.. code-block:: console
55
+
56
+ $ VLLM_TARGET_DEVICE=xpu python setup.py install
57
+
58
+.. note::
59
+ - FP16 is the default data type in the current XPU backend. The BF16 data
60
+ type will be supported in the future.
61
+
docs/source/index.rst
+1
@@ -66,6 +66,7 @@ Documentation
66
getting_started/cpu-installation
67
getting_started/neuron-installation
68
getting_started/tpu-installation
69
+ getting_started/xpu-installation
70
getting_started/quickstart
71
getting_started/debugging
72
getting_started/examples/examples_index
requirements-xpu.txt
new
+11
@@ -0,0 +1,11 @@
1
+# Common dependencies
2
+-r requirements-common.txt
3
+
4
+setuptools < 70.0.0 # IPEX's torch have some dependency. to be removed.
5
+
6
+torch @ https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/xpu/torch-2.1.0.post1%2Bcxx11.abi-cp310-cp310-linux_x86_64.whl
7
+intel_extension_for_pytorch @ https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.1.30a0-cp310-cp310-linux_x86_64.whl
8
+oneccl_bind_pt @ https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/xpu/oneccl_bind_pt-2.1.200%2Bxpu-cp310-cp310-linux_x86_64.whl
9
+
10
+triton @ https://github.com/intel/intel-xpu-backend-for-triton/releases/download/v2.1.0/triton-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
11
+
setup.py
+8
@@ -233,6 +233,10 @@ def _is_cpu() -> bool:
233
return VLLM_TARGET_DEVICE == "cpu"
234
235
236
+def _is_xpu() -> bool:
237
+ return VLLM_TARGET_DEVICE == "xpu"
238
+
239
+
240
def _build_custom_ops() -> bool:
241
return _is_cuda() or _is_hip() or _is_cpu()
242
@@ -337,6 +341,8 @@ def get_vllm_version() -> str:
341
version += "+tpu"
342
elif _is_cpu():
343
version += "+cpu"
344
+ elif _is_xpu():
345
+ version += "+xpu"
346
else:
347
raise RuntimeError("Unknown runtime environment")
348
@@ -386,6 +392,8 @@ def get_requirements() -> List[str]:
392
requirements = _read_requirements("requirements-tpu.txt")
393
elif _is_cpu():
394
requirements = _read_requirements("requirements-cpu.txt")
395
+ elif _is_xpu():
396
+ requirements = _read_requirements("requirements-xpu.txt")
397
else:
398
raise ValueError(
399
"Unsupported platform, please use CUDA, ROCm, Neuron, or CPU.")
vllm/_custom_ops.py
+2
-1
@@ -373,7 +373,8 @@ def reshape_and_cache_flash(
373
kv_cache_dtype)
374
375
376
-def copy_blocks(key_caches: torch.Tensor, value_caches: torch.Tensor,
376
+def copy_blocks(key_caches: List[torch.Tensor],
377
+ value_caches: List[torch.Tensor],
378
block_mapping: torch.Tensor) -> None:
379
torch.ops._C_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
380
vllm/_ipex_ops.py
new
+241
@@ -0,0 +1,241 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.logger import init_logger
6
+
7
+logger = init_logger(__name__)
8
+
9
+try:
10
+ import intel_extension_for_pytorch as ipex
11
+except ImportError as e:
12
+ logger.warning("Import error msg: %s", e.msg)
13
+
14
+
15
+class ipex_ops:
16
+
17
+ @staticmethod
18
+ def _reshape_activation_tensor(
19
+ x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
20
+ num = x.size(0)
21
+ d = x.size(1) // 2
22
+ x = x.reshape(num, 2, d)
23
+ x1, x2 = torch.chunk(x, chunks=2, dim=1)
24
+ x1 = x1.reshape(num, d)
25
+ x2 = x2.reshape(num, d)
26
+ return x1, x2
27
+
28
+ def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
29
+ x1, x2 = ipex_ops._reshape_activation_tensor(x)
30
+ ipex.llm.functional.silu_mul(x1, x2, out)
31
+
32
+ def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
33
+ x1, x2 = ipex_ops._reshape_activation_tensor(x)
34
+ ipex.llm.functional.gelu_mul(x1, x2, out, "none")
35
+
36
+ def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
37
+ x1, x2 = ipex_ops._reshape_activation_tensor(x)
38
+ ipex.llm.functional.gelu_mul(x1, x2, out, "tanh")
39
+
40
+ def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
41
+ out.copy_(torch.nn.functional.gelu(x))
42
+
43
+ def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
44
+ out.copy_(torch.nn.functional.gelu(x))
45
+
46
+ def paged_attention_v1(
47
+ out: torch.Tensor,
48
+ query: torch.Tensor,
49
+ key_cache: torch.Tensor,
50
+ value_cache: torch.Tensor,
51
+ num_kv_heads: int,
52
+ scale: float,
53
+ block_tables: torch.Tensor,
54
+ context_lens: torch.Tensor,
55
+ block_size: int,
56
+ max_context_len: int,
57
+ alibi_slopes: Optional[torch.Tensor],
58
+ kv_cache_dtype: str,
59
+ kv_scale: float,
60
+ tp_rank: int = 0,
61
+ blocksparse_local_blocks: int = 0,
62
+ blocksparse_vert_stride: int = 0,
63
+ blocksparse_block_size: int = 64,
64
+ blocksparse_head_sliding_step: int = 0,
65
+ ) -> None:
66
+ assert kv_cache_dtype == "auto"
67
+ num_heads = out.size(1)
68
+ num_queries_per_tokens = num_heads // num_kv_heads
69
+ head_mapping = torch.arange(
70
+ 0,
71
+ num_kv_heads,
72
+ device=query.device,
73
+ dtype=torch.int32,
74
+ ).view(num_kv_heads,
75
+ 1).repeat_interleave(num_queries_per_tokens).flatten()
76
+ # todo: ipex will refactor namespace
77
+ torch.xpu.paged_attention_v1(out, query.contiguous(),
78
+ key_cache.view_as(value_cache),
79
+ value_cache, head_mapping, scale,
80
+ block_tables, context_lens, block_size,
81
+ max_context_len, alibi_slopes)
82
+
83
+ def paged_attention_v2(
84
+ out: torch.Tensor,
85
+ exp_sum: torch.Tensor,
86
+ max_logits: torch.Tensor,
87
+ tmp_out: torch.Tensor,
88
+ query: torch.Tensor,
89
+ key_cache: torch.Tensor,
90
+ value_cache: torch.Tensor,
91
+ num_kv_heads: int,
92
+ scale: float,
93
+ block_tables: torch.Tensor,
94
+ context_lens: torch.Tensor,
95
+ block_size: int,
96
+ max_context_len: int,
97
+ alibi_slopes: Optional[torch.Tensor],
98
+ kv_cache_dtype: str,
99
+ kv_scale: float,
100
+ tp_rank: int = 0,
101
+ blocksparse_local_blocks: int = 0,
102
+ blocksparse_vert_stride: int = 0,
103
+ blocksparse_block_size: int = 64,
104
+ blocksparse_head_sliding_step: int = 0,
105
+ ) -> None:
106
+ assert kv_cache_dtype == "auto"
107
+ num_heads = out.size(1)
108
+ num_queries_per_tokens = num_heads // num_kv_heads
109
+ head_mapping = torch.arange(
110
+ 0,
111
+ num_kv_heads,
112
+ dtype=torch.int32,
113
+ device=query.device,
114
+ ).view(num_kv_heads,
115
+ 1).repeat_interleave(num_queries_per_tokens).flatten()
116
+ # todo: ipex will refactor namespace
117
+ torch.xpu.paged_attention_v2(out, exp_sum, max_logits, tmp_out,
118
+ query.contiguous(),
119
+ key_cache.view_as(value_cache),
120
+ value_cache, head_mapping, block_tables,
121
+ context_lens, scale, block_size,
122
+ max_context_len, alibi_slopes)
123
+
124
+ def rotary_embedding(
125
+ positions: torch.Tensor, # [batch_size, seq_len]
126
+ query: torch.Tensor, # [batch_size, seq_len, num_heads*head_size]
127
+ key: torch.Tensor, # [batch_size, seq_len, num_kv_heads*head_size]
128
+ head_size: int,
129
+ cos_sin_cache: torch.Tensor, # [cos_sin_dim, rot_dim]
130
+ is_neox: bool,
131
+ ) -> None:
132
+ if positions.dim() == 1:
133
+ positions = positions.unsqueeze(0)
134
+ query = query.unsqueeze(0)
135
+ key = key.unsqueeze(0)
136
+
137
+ rotary_dim = cos_sin_cache.size(1)
138
+ query = query.view(*query.shape[:-1], -1, head_size)
139
+ key = key.view(*key.shape[:-1], -1, head_size)
140
+
141
+ query_rot = query[..., :rotary_dim]
142
+ key_rot = key[..., :rotary_dim]
143
+
144
+ cos_sin = cos_sin_cache[positions.long()]
145
+ cos, sin = cos_sin.chunk(2, dim=-1)
146
+
147
+ if is_neox:
148
+ cos = cos.repeat(1, 1, 2).unsqueeze(-2)
149
+ sin = sin.repeat(1, 1, 2).unsqueeze(-2)
150
+ else:
151
+ cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
152
+ sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
153
+ ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
154
+ rotary_dim, is_neox, positions)
155
+
156
+ def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
157
+ key: torch.Tensor, head_size: int,
158
+ cos_sin_cache: torch.Tensor, is_neox: bool,
159
+ rot_dim: int,
160
+ cos_sin_cache_offsets: torch.Tensor) -> None:
161
+ if positions.dim() == 1:
162
+ positions = positions.unsqueeze(0)
163
+ query = query.unsqueeze(0)
164
+ key = key.unsqueeze(0)
165
+ cos_sin_cache_offsets = cos_sin_cache_offsets.view_as(positions)
166
+ rotary_dim = cos_sin_cache.size(1)
167
+ query = query.view(*query.shape[:-1], -1, head_size)
168
+ key = key.view(*key.shape[:-1], -1, head_size)
169
+
170
+ query_rot = query[..., :rotary_dim]
171
+ key_rot = key[..., :rotary_dim]
172
+
173
+ cos_sin = cos_sin_cache[torch.add(positions,
174
+ cos_sin_cache_offsets).long()]
175
+ cos, sin = cos_sin.chunk(2, dim=-1)
176
+
177
+ if is_neox:
178
+ cos = cos.repeat(1, 1, 2).unsqueeze(-2)
179
+ sin = sin.repeat(1, 1, 2).unsqueeze(-2)
180
+ else:
181
+ cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
182
+ sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
183
+
184
+ ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
185
+ rotary_dim, is_neox, positions)
186
+
187
+ def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor,
188
+ epsilon: float) -> None:
189
+ tmp = ipex.llm.functional.rms_norm(input, weight, epsilon)
190
+ out.copy_(tmp)
191
+
192
+ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
193
+ weight: torch.Tensor, epsilon: float) -> None:
194
+ tmp = ipex.llm.functional.add_rms_norm(residual, input, weight, None,
195
+ epsilon, True)
196
+ input.copy_(tmp)
197
+
198
+ def varlen_attention(
199
+ query: torch.Tensor,
200
+ key: torch.Tensor,
201
+ value: torch.Tensor,
202
+ out: torch.Tensor,
203
+ seqlen_q: torch.Tensor,
204
+ seqlen_k: torch.Tensor,
205
+ max_seqlen_q: int,
206
+ max_seqlen_k: int,
207
+ pdropout: float,
208
+ softmax_scale: float,
209
+ zero_tensors: bool,
210
+ is_causal: bool,
211
+ return_softmax: bool,
212
+ gen_: torch.Generator,
213
+ ) -> None:
214
+ ipex.llm.functional.varlen_attention(query, key, value, out, seqlen_q,
215
+ seqlen_k, max_seqlen_q,
216
+ max_seqlen_k, pdropout,
217
+ softmax_scale, zero_tensors,
218
+ is_causal, return_softmax, gen_)
219
+
220
+ def reshape_and_cache(
221
+ key: torch.Tensor,
222
+ value: torch.Tensor,
223
+ key_cache: torch.Tensor,
224
+ value_cache: torch.Tensor,
225
+ slot_mapping: torch.Tensor,
226
+ kv_cache_dtype: str,
227
+ kv_scale: float,
228
+ ) -> None:
229
+ assert kv_cache_dtype == "auto"
230
+ ipex.llm.modules.PagedAttention.reshape_and_cache(
231
+ key, value, key_cache, value_cache, slot_mapping)
232
+
233
+ @staticmethod
234
+ def copy_blocks(key_caches: List[torch.Tensor],
235
+ value_caches: List[torch.Tensor],
236
+ block_mapping: torch.Tensor) -> None:
237
+ torch.xpu.copy_blocks(key_caches, value_caches, block_mapping)
238
+
239
+ def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
240
+ block_mapping: torch.Tensor) -> None:
241
+ torch.xpu.swap_blocks(src, dst, block_mapping)
vllm/attention/backends/ipex_attn.py
new
+355
@@ -0,0 +1,355 @@
1
+""" Attention layer with torch scaled_dot_product_attention
2
+ and PagedAttention."""
3
+from dataclasses import dataclass
4
+from typing import Any, Dict, List, Optional, Tuple, Type
5
+
6
+import torch
7
+
8
+from vllm._ipex_ops import ipex_ops
9
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
+ AttentionMetadata)
11
+from vllm.attention.ops.paged_attn import (PagedAttention,
12
+ PagedAttentionMetadata)
13
+
14
+_PARTITION_SIZE = 512
15
+
16
+
17
+class IpexAttnBackend(AttentionBackend):
18
+
19
+ @staticmethod
20
+ def get_name() -> str:
21
+ return "ipex-attn"
22
+
23
+ @staticmethod
24
+ def get_impl_cls() -> Type["IpexAttnBackendImpl"]:
25
+ return IpexAttnBackendImpl
26
+
27
+ @staticmethod
28
+ def make_metadata(*args, **kwargs) -> "IpexAttnMetadata":
29
+ return IpexAttnMetadata(*args, **kwargs)
30
+
31
+ @staticmethod
32
+ def get_kv_cache_shape(
33
+ num_blocks: int,
34
+ block_size: int,
35
+ num_kv_heads: int,
36
+ head_size: int,
37
+ ) -> Tuple[int, ...]:
38
+ return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
39
+ num_kv_heads, head_size)
40
+
41
+ @staticmethod
42
+ def swap_blocks(
43
+ src_kv_cache: torch.Tensor,
44
+ dst_kv_cache: torch.Tensor,
45
+ src_to_dst: torch.Tensor,
46
+ ) -> None:
47
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
48
+
49
+ @staticmethod
50
+ def copy_blocks(
51
+ kv_caches: List[torch.Tensor],
52
+ src_to_dists: torch.Tensor,
53
+ ) -> None:
54
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
55
+
56
+
57
+@dataclass
58
+class IpexAttnMetadata(AttentionMetadata, PagedAttentionMetadata):
59
+ """Metadata for IpexAttnBackend.
60
+ """
61
+ # Currently, input sequences can only contain all prompts
62
+ # or all decoding. True if all sequences are prompts.
63
+ is_prompt: bool
64
+ slot_mapping: torch.Tensor
65
+ seq_lens: Optional[List[int]]
66
+ seqlen_q: Optional[torch.Tensor]
67
+ max_seqlen: Optional[int]
68
+
69
+ def __post_init__(self):
70
+ # Set during the execution of the first attention op.
71
+ # It is a list because it is needed to set per prompt
72
+ # when alibi slopes is used. It is because of the limitation
73
+ # from xformer API.
74
+ # will not appear in the __repr__ and __init__
75
+ self.attn_bias: Optional[List[torch.Tensor]] = None
76
+
77
+ @property
78
+ def prefill_metadata(self) -> Optional["IpexAttnMetadata"]:
79
+ # Currently chunked prefill is not supported
80
+ if self.num_decode_tokens == 0:
81
+ assert self.num_prefills > 0
82
+ return self
83
+
84
+ return None
85
+
86
+ @property
87
+ def decode_metadata(self) -> Optional["IpexAttnMetadata"]:
88
+ # Currently chunked prefill is not supported
89
+ if self.num_prefills > 0:
90
+ assert self.num_decode_tokens == 0
91
+ return None
92
+
93
+ return self
94
+
95
+
96
+class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
97
+
98
+ def __init__(
99
+ self,
100
+ num_heads: int,
101
+ head_size: int,
102
+ scale: float,
103
+ num_kv_heads: int,
104
+ alibi_slopes: Optional[List[float]],
105
+ sliding_window: Optional[int],
106
+ kv_cache_dtype: str,
107
+ blocksparse_params: Optional[Dict[str, Any]] = None,
108
+ ) -> None:
109
+ assert blocksparse_params is None, ValueError(
110
+ "Torch SPDA does not support block-sparse attention.")
111
+ self.num_heads = num_heads
112
+ self.head_size = head_size
113
+ self.scale = float(scale)
114
+ self.num_kv_heads = num_kv_heads
115
+ if alibi_slopes is not None:
116
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
117
+ self.alibi_slopes = alibi_slopes
118
+ self.sliding_window = sliding_window
119
+ self.kv_cache_dtype = kv_cache_dtype
120
+
121
+ assert self.num_heads % self.num_kv_heads == 0
122
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
123
+ self.need_mask = (self.alibi_slopes is not None
124
+ or self.sliding_window is not None)
125
+
126
+ supported_head_sizes = PagedAttention.get_supported_head_sizes()
127
+ if head_size not in supported_head_sizes:
128
+ raise ValueError(
129
+ f"Head size {head_size} is not supported by PagedAttention. "
130
+ f"Supported head sizes are: {supported_head_sizes}.")
131
+ if kv_cache_dtype != "auto":
132
+ raise NotImplementedError(
133
+ "IPEX backend does not support FP8 KV cache. "
134
+ "Please use xFormers backend instead.")
135
+
136
+ def split_kv_cache(
137
+ self,
138
+ kv_cache: torch.Tensor,
139
+ num_kv_heads: int,
140
+ head_size: int,
141
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
142
+ x = 1
143
+ num_blocks = kv_cache.shape[1]
144
+
145
+ key_cache = kv_cache[0]
146
+ key_cache = key_cache.view(num_blocks, num_kv_heads, head_size // x,
147
+ -1, x)
148
+ value_cache = kv_cache[1]
149
+ value_cache = value_cache.view(num_blocks, num_kv_heads, head_size, -1)
150
+ return key_cache, value_cache
151
+
152
+ def forward(
153
+ self,
154
+ query: torch.Tensor,
155
+ key: torch.Tensor,
156
+ value: torch.Tensor,
157
+ kv_cache: Optional[torch.Tensor],
158
+ attn_metadata: IpexAttnMetadata, # type: ignore
159
+ kv_scale: float = 1.0,
160
+ ) -> torch.Tensor:
161
+ """Forward pass with IPEX varlen_attention and PagedAttention.
162
+
163
+ Args:
164
+ query: shape = [num_tokens, num_heads * head_size]
165
+ key: shape = [num_tokens, num_kv_heads * head_size]
166
+ value: shape = [num_tokens, num_kv_heads * head_size]
167
+ kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
168
+ attn_metadata: Metadata for attention.
169
+ Returns:
170
+ shape = [num_tokens, num_heads * head_size]
171
+ """
172
+ assert kv_scale == 1.0
173
+ num_tokens, hidden_size = query.shape
174
+ # Reshape the query, key, and value tensors.
175
+ query = query.view(-1, self.num_heads, self.head_size)
176
+ key = key.view(-1, self.num_kv_heads, self.head_size)
177
+ value = value.view(-1, self.num_kv_heads, self.head_size)
178
+
179
+ if kv_cache is not None:
180
+ key_cache, value_cache = self.split_kv_cache(
181
+ kv_cache, self.num_kv_heads, self.head_size)
182
+ ipex_ops.reshape_and_cache(
183
+ key,
184
+ value,
185
+ key_cache,
186
+ value_cache,
187
+ attn_metadata.slot_mapping.flatten(),
188
+ self.kv_cache_dtype,
189
+ kv_scale,
190
+ )
191
+
192
+ if attn_metadata.is_prompt:
193
+ assert attn_metadata.seq_lens is not None
194
+ if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
195
+ if self.num_kv_heads != self.num_heads:
196
+ key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
197
+ value = value.repeat_interleave(self.num_queries_per_kv,
198
+ dim=1)
199
+
200
+ if attn_metadata.attn_bias is None:
201
+ if self.alibi_slopes is not None:
202
+ att_masks = _make_alibi_bias(
203
+ self.alibi_slopes, query.dtype,
204
+ attn_metadata.seq_lens) # type: ignore
205
+ elif self.sliding_window is not None:
206
+ att_masks = _make_sliding_window_bias(
207
+ attn_metadata.seq_lens, self.sliding_window,
208
+ query.dtype) # type: ignore
209
+ else:
210
+ att_masks = _make_sliding_window_bias(
211
+ attn_metadata.seq_lens, None, dtype=query.dtype)
212
+ attn_metadata.attn_bias = att_masks
213
+
214
+ output = torch.empty(
215
+ (num_tokens, self.num_heads, self.head_size),
216
+ dtype=query.dtype,
217
+ device=query.device)
218
+ ipex_ops.varlen_attention(query,
219
+ key,
220
+ value,
221
+ output,
222
+ attn_metadata.seqlen_q,
223
+ attn_metadata.seqlen_q,
224
+ attn_metadata.max_seqlen,
225
+ attn_metadata.max_seqlen,
226
+ pdropout=0.0,
227
+ softmax_scale=self.scale,
228
+ zero_tensors=False,
229
+ is_causal=True,
230
+ return_softmax=False,
231
+ gen_=None)
232
+ else:
233
+ # prefix-enabled attention
234
+ raise RuntimeError(
235
+ "IPEX backend doesn't support prefix decoding.")
236
+
237
+ else:
238
+ # Decoding run.
239
+ max_seq_len = attn_metadata.max_decode_seq_len
240
+ output = torch.empty_like(query)
241
+ block_size = value_cache.shape[3]
242
+ num_seqs, num_heads, head_size = query.shape
243
+ max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
244
+ _PARTITION_SIZE)
245
+ # NOTE(woosuk): We use a simple heuristic to decide whether to use
246
+ # PagedAttention V1 or V2. If the number of partitions is 1, we use
247
+ # V1 to avoid the overhead of reduction. Also, if the number of
248
+ # sequences or heads is large, we use V1 since there is enough work
249
+ # to parallelize.
250
+ # TODO(woosuk): Tune this heuristic.
251
+ # For context len > 8192, use V2 kernel to avoid shared memory
252
+ # shortage.
253
+ use_v1 = (max_seq_len <= 8192 and
254
+ (max_num_partitions == 1 or num_seqs * num_heads > 512))
255
+ if use_v1:
256
+ # Run PagedAttention V1.
257
+ ipex_ops.paged_attention_v1(
258
+ output,
259
+ query,
260
+ key_cache,
261
+ value_cache,
262
+ self.num_kv_heads,
263
+ self.scale,
264
+ attn_metadata.block_tables,
265
+ attn_metadata.seq_lens_tensor,
266
+ block_size,
267
+ max_seq_len,
268
+ self.alibi_slopes,
269
+ self.kv_cache_dtype,
270
+ kv_scale,
271
+ )
272
+ else:
273
+ # Run PagedAttention V2.
274
+ assert _PARTITION_SIZE % block_size == 0
275
+ tmp_output = torch.empty(
276
+ size=(num_seqs, num_heads, max_num_partitions, head_size),
277
+ dtype=output.dtype,
278
+ device=output.device,
279
+ )
280
+ exp_sums = torch.empty(
281
+ size=(num_seqs, num_heads, max_num_partitions),
282
+ dtype=torch.float32,
283
+ device=output.device,
284
+ )
285
+ max_logits = torch.empty_like(exp_sums)
286
+ ipex_ops.paged_attention_v2(
287
+ output,
288
+ exp_sums,
289
+ max_logits,
290
+ tmp_output,
291
+ query,
292
+ key_cache,
293
+ value_cache,
294
+ self.num_kv_heads,
295
+ self.scale,
296
+ attn_metadata.block_tables,
297
+ attn_metadata.seq_lens_tensor,
298
+ block_size,
299
+ max_seq_len,
300
+ self.alibi_slopes,
301
+ self.kv_cache_dtype,
302
+ kv_scale,
303
+ )
304
+
305
+ # Reshape the output tensor.
306
+ return output.view(-1, self.num_heads * self.head_size)
307
+
308
+
309
+def _make_alibi_bias(
310
+ alibi_slopes: torch.Tensor,
311
+ dtype: torch.dtype,
312
+ seq_lens: List[int],
313
+) -> List[torch.Tensor]:
314
+ attn_biases = []
315
+ for seq_len in seq_lens:
316
+ bias = torch.arange(seq_len, dtype=dtype, device=alibi_slopes.device)
317
+ # NOTE(zhuohan): HF uses
318
+ # `bias = bias[None, :].repeat(seq_len, 1)`
319
+ # here. We find that both biases give the same results, but
320
+ # the bias below more accurately follows the original ALiBi
321
+ # paper.
322
+ bias = bias[None, :] - bias[:, None]
323
+
324
+ num_heads = alibi_slopes.shape[0]
325
+ bias = bias[None, :].repeat((num_heads, 1, 1))
326
+ bias.mul_(alibi_slopes[:, None, None])
327
+ inf_mask = torch.empty(
328
+ (1, seq_len, seq_len),
329
+ dtype=bias.dtype,
330
+ device=alibi_slopes.device).fill_(-torch.inf).triu_(diagonal=1)
331
+ attn_biases.append((bias + inf_mask).to(dtype))
332
+
333
+ return attn_biases
334
+
335
+
336
+def _make_sliding_window_bias(
337
+ seq_lens: List[int],
338
+ window_size: Optional[int],
339
+ dtype: torch.dtype,
340
+) -> List[torch.Tensor]:
341
+ attn_biases = []
342
+ for seq_len in seq_lens:
343
+ tensor = torch.full(
344
+ (1, seq_len, seq_len),
345
+ dtype=dtype,
346
+ fill_value=1,
347
+ )
348
+ shift = 0
349
+ mask = torch.tril(tensor, diagonal=shift).to(dtype) # type: ignore
350
+ if window_size is not None:
351
+ mask = torch.triu(mask, diagonal=shift - window_size + 1)
352
+ mask = torch.log(mask)
353
+ attn_biases.append(mask.to(dtype))
354
+
355
+ return attn_biases
vllm/attention/selector.py
+13
-2
@@ -7,7 +7,7 @@ import torch
7
import vllm.envs as envs
8
from vllm.attention.backends.abstract import AttentionBackend
9
from vllm.logger import init_logger
10
-from vllm.utils import is_cpu, is_hip, is_tpu
10
+from vllm.utils import is_cpu, is_hip, is_tpu, is_xpu
11
12
logger = init_logger(__name__)
13
@@ -19,6 +19,7 @@ class _Backend(enum.Enum):
19
TORCH_SDPA = enum.auto()
20
FLASHINFER = enum.auto()
21
PALLAS = enum.auto()
22
+ IPEX = enum.auto()
23
24
25
@lru_cache(maxsize=None)
@@ -58,12 +59,17 @@ def get_attn_backend(
59
ROCmFlashAttentionBackend)
60
return ROCmFlashAttentionBackend
61
elif backend == _Backend.TORCH_SDPA:
61
- # TODO: make XPU backend available here.
62
assert is_cpu(), RuntimeError(
63
"Torch SDPA backend is only used for the CPU device.")
64
logger.info("Using Torch SDPA backend.")
65
from vllm.attention.backends.torch_sdpa import TorchSDPABackend
66
return TorchSDPABackend
67
+ elif backend == _Backend.IPEX:
68
+ assert is_xpu(), RuntimeError(
69
+ "IPEX attention backend is only used for the XPU device.")
70
+ logger.info("Using IPEX attention backend.")
71
+ from vllm.attention.backends.ipex_attn import IpexAttnBackend
72
+ return IpexAttnBackend
73
elif backend == _Backend.FLASHINFER:
74
logger.info("Using Flashinfer backend.")
75
logger.warning("Eager mode is required for the Flashinfer backend. "
@@ -107,6 +113,11 @@ def which_attn_to_use(
113
logger.info("Cannot use %s backend on CPU.", selected_backend)
114
return _Backend.TORCH_SDPA
115
116
+ if is_xpu():
117
+ if selected_backend != _Backend.IPEX:
118
+ logger.info("Cannot use %s backend on XPU.", selected_backend)
119
+ return _Backend.IPEX
120
+
121
if is_tpu():
122
if selected_backend != _Backend.PALLAS:
123
logger.info("Cannot use %s backend on TPU.", selected_backend)
vllm/config.py
+3
-1
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.model_executor.models import ModelRegistry
13
from vllm.transformers_utils.config import get_config, get_hf_text_config
14
from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
15
- is_hip, is_neuron, is_tpu)
15
+ is_hip, is_neuron, is_tpu, is_xpu)
16
17
if TYPE_CHECKING:
18
from ray.util.placement_group import PlacementGroup
@@ -757,6 +757,8 @@ class DeviceConfig:
757
self.device_type = "tpu"
758
elif is_cpu():
759
self.device_type = "cpu"
760
+ elif is_xpu():
761
+ self.device_type = "xpu"
762
else:
763
# We don't call torch.cuda.is_available() here to
764
# avoid initializing CUDA before workers are forked
vllm/distributed/parallel_state.py
+1
-1
@@ -58,7 +58,7 @@ def _split_tensor_dict(
58
# because it contains not only the device type but also the device
59
# index (e.g. "cuda:0"). We only need the device type.
60
# receiving side will set the device index.
61
- device = "cpu" if value.is_cpu else "cuda"
61
+ device = value.device.type
62
metadata_list.append(
63
(key, TensorMetadata(device, value.dtype, value.size())))
64
tensor_list.append(value)
vllm/engine/arg_utils.py
+6
-5
@@ -501,11 +501,12 @@ class EngineArgs:
501
'Enabling this will use the fully sharded layers. '
502
'At high sequence length, max rank or '
503
'tensor parallel size, this is likely faster.'))
504
- parser.add_argument("--device",
505
- type=str,
506
- default=EngineArgs.device,
507
- choices=["auto", "cuda", "neuron", "cpu", "tpu"],
508
- help='Device type for vLLM execution.')
504
+ parser.add_argument(
505
+ "--device",
506
+ type=str,
507
+ default=EngineArgs.device,
508
+ choices=["auto", "cuda", "neuron", "cpu", "tpu", "xpu"],
509
+ help='Device type for vLLM execution.')
510
511
# Related to Vision-language models such as llava
512
parser = EngineArgs.add_cli_args_for_vlm(parser)
vllm/engine/async_llm_engine.py
+11
@@ -383,6 +383,17 @@ class AsyncLLMEngine:
383
"Distributed execution is not supported with the CPU backend.")
384
from vllm.executor.cpu_executor import CPUExecutorAsync
385
executor_class = CPUExecutorAsync
386
+ elif engine_config.device_config.device_type == "xpu":
387
+ if distributed_executor_backend is None:
388
+ from vllm.executor.xpu_executor import XPUExecutorAsync
389
+ executor_class = XPUExecutorAsync
390
+ elif distributed_executor_backend == "ray":
391
+ initialize_ray_cluster(engine_config.parallel_config)
392
+ from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync
393
+ executor_class = RayXPUExecutorAsync
394
+ else:
395
+ raise RuntimeError(
396
+ "Not supported distributed execution model on XPU device.")
397
elif distributed_executor_backend == "ray":
398
initialize_ray_cluster(engine_config.parallel_config)
399
from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
vllm/engine/llm_engine.py
+8
@@ -347,6 +347,14 @@ class LLMEngine:
347
elif engine_config.device_config.device_type == "cpu":
348
from vllm.executor.cpu_executor import CPUExecutor
349
executor_class = CPUExecutor
350
+ elif engine_config.device_config.device_type == "xpu":
351
+ if distributed_executor_backend == "ray":
352
+ initialize_ray_cluster(engine_config.parallel_config)
353
+ from vllm.executor.ray_xpu_executor import RayXPUExecutor
354
+ executor_class = RayXPUExecutor
355
+ else:
356
+ from vllm.executor.xpu_executor import XPUExecutor
357
+ executor_class = XPUExecutor
358
elif distributed_executor_backend == "ray":
359
initialize_ray_cluster(engine_config.parallel_config)
360
from vllm.executor.ray_gpu_executor import RayGPUExecutor
vllm/executor/ray_utils.py
+2
-2
@@ -3,7 +3,7 @@ from typing import List, Optional, Tuple
3
4
from vllm.config import ParallelConfig
5
from vllm.logger import init_logger
6
-from vllm.utils import get_ip, is_hip
6
+from vllm.utils import get_ip, is_hip, is_xpu
7
from vllm.worker.worker_base import WorkerWrapperBase
8
9
logger = init_logger(__name__)
@@ -71,7 +71,7 @@ def initialize_ray_cluster(
71
"serving.")
72
73
# Connect to a ray cluster.
74
- if is_hip():
74
+ if is_hip() or is_xpu():
75
ray.init(address=ray_address,
76
ignore_reinit_error=True,
77
num_gpus=parallel_config.world_size)
vllm/executor/ray_xpu_executor.py
new
+401
@@ -0,0 +1,401 @@
1
+import asyncio
2
+import os
3
+import pickle
4
+from collections import defaultdict
5
+from itertools import islice, repeat
6
+from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
7
+ Tuple, Union)
8
+
9
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
+ ModelConfig, ParallelConfig, SchedulerConfig,
11
+ SpeculativeConfig, VisionLanguageConfig)
12
+from vllm.executor.distributed_gpu_executor import ( # yapf: disable
13
+ DistributedGPUExecutor, DistributedGPUExecutorAsync)
14
+from vllm.executor.ray_utils import RayWorkerWrapper, ray
15
+from vllm.logger import init_logger
16
+from vllm.lora.request import LoRARequest
17
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
18
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
19
+ make_async)
20
+
21
+if ray is not None:
22
+ from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
23
+
24
+if TYPE_CHECKING:
25
+ from ray.util.placement_group import PlacementGroup
26
+
27
+logger = init_logger(__name__)
28
+
29
+# If the env var is set, it uses the Ray's compiled DAG API
30
+# which optimizes the control plane overhead.
31
+# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
32
+USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
33
+
34
+
35
+class RayXPUExecutor(DistributedGPUExecutor):
36
+
37
+ def __init__(
38
+ self,
39
+ model_config: ModelConfig,
40
+ cache_config: CacheConfig,
41
+ parallel_config: ParallelConfig,
42
+ scheduler_config: SchedulerConfig,
43
+ device_config: DeviceConfig,
44
+ load_config: LoadConfig,
45
+ lora_config: Optional[LoRAConfig],
46
+ vision_language_config: Optional[VisionLanguageConfig],
47
+ speculative_config: Optional[SpeculativeConfig],
48
+ ) -> None:
49
+ assert device_config.device_type == "xpu"
50
+ assert (not speculative_config
51
+ ), "Speculative decoding not yet supported for XPU backend"
52
+
53
+ self.model_config = model_config
54
+ self.cache_config = cache_config
55
+ self.load_config = load_config
56
+ self.lora_config = lora_config
57
+ self.parallel_config = parallel_config
58
+ self.scheduler_config = scheduler_config
59
+ self.device_config = device_config
60
+ self.vision_language_config = vision_language_config
61
+
62
+ placement_group = self.parallel_config.placement_group
63
+
64
+ # Disable Ray usage stats collection.
65
+ ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
66
+ if ray_usage != "1":
67
+ os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
68
+
69
+ # Create the parallel GPU workers.
70
+ self._init_workers_ray(placement_group)
71
+
72
+ # Profile the memory usage and initialize the cache.
73
+ self.forward_dag = None
74
+ if USE_RAY_COMPILED_DAG:
75
+ self.forward_dag = self._compiled_ray_dag()
76
+
77
+ # This is non-None when the execute model loop is running
78
+ # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
79
+ self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
80
+ # Updated by implementations that require additional args to be passed
81
+ # to the _run_workers execute_model call
82
+ self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
83
+
84
+ def _init_executor(self) -> None:
85
+ pass
86
+
87
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
88
+ """Determine the number of available KV blocks.
89
+
90
+ This invokes `determine_num_available_blocks` on each worker and takes
91
+ the min of the results, guaranteeing that the selected cache sizes are
92
+ compatible with all workers.
93
+
94
+ Returns:
95
+ - Tuple[num_gpu_blocks, num_cpu_blocks]
96
+ """
97
+ # Get the maximum number of blocks that can be allocated on GPU and CPU.
98
+ num_blocks = self._run_workers("determine_num_available_blocks", )
99
+
100
+ # Since we use a shared centralized controller, we take the minimum
101
+ # number of blocks across all workers to make sure all the memory
102
+ # operators can be applied to all workers.
103
+ num_gpu_blocks = min(b[0] for b in num_blocks)
104
+ num_cpu_blocks = min(b[1] for b in num_blocks)
105
+
106
+ return num_gpu_blocks, num_cpu_blocks
107
+
108
+ def _init_workers_ray(self, placement_group: "PlacementGroup",
109
+ **ray_remote_kwargs):
110
+ if self.parallel_config.tensor_parallel_size == 1:
111
+ # For single GPU case, we use a ray worker with constrained memory.
112
+ num_gpus = self.cache_config.gpu_memory_utilization
113
+ else:
114
+ # Otherwise, the ray workers are allocated with a full GPU.
115
+ num_gpus = 1
116
+
117
+ # The driver dummy worker does not actually use any resources.
118
+ # It holds the resource for the driver worker.
119
+ self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
120
+ # The remaining workers are the actual ray actors.
121
+ self.workers: List[RayWorkerWrapper] = []
122
+
123
+ # Create the workers.
124
+ driver_ip = get_ip()
125
+ for bundle_id, bundle in enumerate(placement_group.bundle_specs):
126
+ if not bundle.get("GPU", 0):
127
+ continue
128
+ scheduling_strategy = PlacementGroupSchedulingStrategy(
129
+ placement_group=placement_group,
130
+ placement_group_capture_child_tasks=True,
131
+ placement_group_bundle_index=bundle_id,
132
+ )
133
+ worker = ray.remote(
134
+ num_cpus=0,
135
+ num_gpus=num_gpus,
136
+ scheduling_strategy=scheduling_strategy,
137
+ **ray_remote_kwargs,
138
+ )(RayWorkerWrapper).remote(
139
+ worker_module_name="vllm.worker.xpu_worker",
140
+ worker_class_name="XPUWorker",
141
+ trust_remote_code=self.model_config.trust_remote_code,
142
+ )
143
+
144
+ worker_ip = ray.get(worker.get_node_ip.remote())
145
+ if worker_ip == driver_ip and self.driver_dummy_worker is None:
146
+ # If the worker is on the same node as the driver, we use it
147
+ # as the resource holder for the driver process.
148
+ self.driver_dummy_worker = worker
149
+ self.driver_worker = RayWorkerWrapper(
150
+ worker_module_name="vllm.worker.xpu_worker",
151
+ worker_class_name="XPUWorker",
152
+ trust_remote_code=self.model_config.trust_remote_code,
153
+ )
154
+ else:
155
+ # Else, added to the list of workers.
156
+ self.workers.append(worker)
157
+ if self.driver_dummy_worker is None:
158
+ raise ValueError(
159
+ "Ray does not allocate any GPUs on the driver node. Consider "
160
+ "adjusting the Ray placement group or running the driver on a "
161
+ "GPU node.")
162
+
163
+ # Get the set of GPU IDs used on each node.
164
+ worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
165
+ use_dummy_driver=True)
166
+
167
+ node_workers = defaultdict(list)
168
+ node_gpus = defaultdict(list)
169
+
170
+ for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
171
+ node_workers[node_id].append(i)
172
+ node_gpus[node_id].extend(gpu_ids)
173
+ for node_id, gpu_ids in node_gpus.items():
174
+ node_gpus[node_id] = sorted(gpu_ids)
175
+
176
+ # TODO: add env var for xpu
177
+
178
+ distributed_init_method = get_distributed_init_method(
179
+ driver_ip, get_open_port())
180
+
181
+ def collect_arg_helper_func(**kwargs):
182
+ # avoid writing `{"name": value}` manually
183
+ return kwargs
184
+
185
+ init_worker_all_kwargs = []
186
+
187
+ # Initialize the actual workers inside worker wrapper.
188
+ for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids, ):
189
+ local_rank = node_workers[node_id].index(rank)
190
+ init_worker_all_kwargs.append(
191
+ collect_arg_helper_func(
192
+ model_config=self.model_config,
193
+ parallel_config=self.parallel_config,
194
+ scheduler_config=self.scheduler_config,
195
+ device_config=self.device_config,
196
+ cache_config=self.cache_config,
197
+ load_config=self.load_config,
198
+ local_rank=local_rank,
199
+ rank=rank,
200
+ distributed_init_method=distributed_init_method,
201
+ lora_config=self.lora_config,
202
+ vision_language_config=self.vision_language_config,
203
+ is_driver_worker=rank == 0,
204
+ ))
205
+ self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
206
+
207
+ self._run_workers("init_device")
208
+ self._run_workers(
209
+ "load_model",
210
+ max_concurrent_workers=self.parallel_config.
211
+ max_parallel_loading_workers,
212
+ )
213
+
214
+ def initialize_cache(self, num_gpu_blocks: int,
215
+ num_cpu_blocks: int) -> None:
216
+ """Initialize the KV cache in all workers.
217
+ """
218
+
219
+ # NOTE: We log here to avoid multiple logs when number of workers is
220
+ # greater than one. We could log in the engine, but not all executors
221
+ # have GPUs.
222
+ logger.info("# GPU blocks: %d, "
223
+ "# CPU blocks: %d", num_gpu_blocks, num_cpu_blocks)
224
+
225
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
226
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
227
+
228
+ self._run_workers("initialize_cache",
229
+ num_gpu_blocks=num_gpu_blocks,
230
+ num_cpu_blocks=num_cpu_blocks)
231
+
232
+ def _driver_execute_model(
233
+ self,
234
+ execute_model_req: Optional[ExecuteModelRequest] = None
235
+ ) -> List[SamplerOutput]:
236
+ """Run execute_model in the driver worker.
237
+
238
+ Passing None will cause the driver to stop the model execution
239
+ loop running in each of the remote workers.
240
+ """
241
+ return self.driver_worker.execute_method("execute_model",
242
+ execute_model_req)
243
+
244
+ def add_lora(self, lora_request: LoRARequest) -> bool:
245
+ assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
246
+ return self._run_workers(
247
+ "add_lora",
248
+ lora_request=lora_request,
249
+ )
250
+
251
+ def remove_lora(self, lora_id: int) -> bool:
252
+ assert lora_id > 0, "lora_id must be greater than 0."
253
+ return self._run_workers(
254
+ "remove_lora",
255
+ lora_id=lora_id,
256
+ )
257
+
258
+ def list_loras(self) -> Set[int]:
259
+ return self._run_workers("list_loras")
260
+
261
+ def _run_workers(
262
+ self,
263
+ method: str,
264
+ *args,
265
+ async_run_remote_workers_only: bool = False,
266
+ all_args: Optional[List[Tuple[Any, ...]]] = None,
267
+ all_kwargs: Optional[List[Dict[str, Any]]] = None,
268
+ use_dummy_driver: bool = False,
269
+ max_concurrent_workers: Optional[int] = None,
270
+ use_ray_compiled_dag: bool = False,
271
+ **kwargs,
272
+ ) -> Any:
273
+ """Runs the given method on all workers. Can be used in the following
274
+ ways:
275
+
276
+ - args/kwargs: All workers share the same args/kwargs
277
+ - args/kwargs and driver_args/driver_kwargs: Driver worker has
278
+ different args
279
+ - all_args/all_kwargs: args/kwargs for each worker are specified
280
+ individually
281
+ """
282
+
283
+ if max_concurrent_workers:
284
+ raise NotImplementedError(
285
+ "max_concurrent_workers is not supported yet.")
286
+
287
+ count = len(self.workers)
288
+ all_worker_args = repeat(args, count) if all_args is None \
289
+ else islice(all_args, 1, None)
290
+ all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
291
+ else islice(all_kwargs, 1, None)
292
+
293
+ if use_ray_compiled_dag:
294
+ # Right now, compiled DAG can only accept a single
295
+ # input. TODO(sang): Fix it.
296
+ assert self.forward_dag is not None
297
+ output_channels = self.forward_dag.execute(1)
298
+ else:
299
+ # Start the ray workers first.
300
+ ray_worker_outputs = [
301
+ worker.execute_method.remote(method, *worker_args,
302
+ **worker_kwargs)
303
+ for (worker, worker_args, worker_kwargs
304
+ ) in zip(self.workers, all_worker_args, all_worker_kwargs)
305
+ ]
306
+ if async_run_remote_workers_only:
307
+ # Just return futures
308
+ return ray_worker_outputs
309
+
310
+ driver_args = args if all_args is None else all_args[0]
311
+ driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
312
+
313
+ # Start the driver worker after all the ray workers.
314
+ if not use_dummy_driver:
315
+ driver_worker_output = self.driver_worker.execute_method(
316
+ method, *driver_args, **driver_kwargs)
317
+ else:
318
+ assert self.driver_dummy_worker is not None
319
+ driver_worker_output = ray.get(
320
+ self.driver_dummy_worker.execute_method.remote(
321
+ method, *driver_args, **driver_kwargs))
322
+ # Get the results of the ray workers.
323
+ if self.workers:
324
+ if use_ray_compiled_dag:
325
+ try:
326
+ ray_worker_outputs = [
327
+ pickle.loads(chan.begin_read())
328
+ for chan in output_channels
329
+ ]
330
+ finally:
331
+ # Has to call end_read in order to reuse the DAG.
332
+ for chan in output_channels:
333
+ chan.end_read()
334
+ else:
335
+ ray_worker_outputs = ray.get(ray_worker_outputs)
336
+
337
+ return [driver_worker_output] + ray_worker_outputs
338
+
339
+ def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
340
+ """Wait for futures returned from _run_workers() with
341
+ async_run_remote_workers_only to complete."""
342
+ ray.get(parallel_worker_tasks)
343
+
344
+ def _compiled_ray_dag(self):
345
+ import pkg_resources
346
+ required_version = "2.9"
347
+ current_version = pkg_resources.get_distribution("ray").version
348
+ if current_version < required_version:
349
+ raise ValueError(f"Ray version {required_version} or greater is "
350
+ f"required, but found {current_version}")
351
+
352
+ from ray.dag import InputNode, MultiOutputNode
353
+ assert self.parallel_config.worker_use_ray
354
+
355
+ # Right now, compiled DAG requires at least 1 arg. We send
356
+ # a dummy value for now. It will be fixed soon.
357
+ with InputNode() as input_data:
358
+ forward_dag = MultiOutputNode([
359
+ worker.execute_model_compiled_dag_remote.
360
+ bind( # type: ignore[attr-defined]
361
+ input_data) for worker in self.workers
362
+ ])
363
+ return forward_dag.experimental_compile()
364
+
365
+ def check_health(self) -> None:
366
+ """Raises an error if engine is unhealthy."""
367
+ self._check_if_any_actor_is_dead()
368
+
369
+ def _check_if_any_actor_is_dead(self):
370
+ if not self.workers:
371
+ return
372
+
373
+ dead_actors = []
374
+ for actor in self.workers:
375
+ actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
376
+ if actor_state["State"] == "DEAD":
377
+ dead_actors.append(actor)
378
+ if dead_actors:
379
+ raise RuntimeError("At least one Worker is dead. "
380
+ f"Dead Workers: {dead_actors}. ")
381
+
382
+
383
+class RayXPUExecutorAsync(RayXPUExecutor, DistributedGPUExecutorAsync):
384
+
385
+ def __init__(self, *args, **kwargs):
386
+ super().__init__(*args, **kwargs)
387
+ self.driver_exec_method = make_async(self.driver_worker.execute_method)
388
+
389
+ async def _driver_execute_model_async(
390
+ self,
391
+ execute_model_req: Optional[ExecuteModelRequest] = None
392
+ ) -> List[SamplerOutput]:
393
+ return await self.driver_exec_method("execute_model",
394
+ execute_model_req)
395
+
396
+ async def _start_worker_execution_loop(self):
397
+ coros = [
398
+ worker.execute_method.remote("start_worker_execution_loop")
399
+ for worker in self.workers
400
+ ]
401
+ return await asyncio.gather(*coros)
vllm/executor/xpu_executor.py
new
+98
@@ -0,0 +1,98 @@
1
+from typing import List, Optional
2
+
3
+import torch
4
+
5
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
+ ModelConfig, ParallelConfig, SchedulerConfig,
7
+ SpeculativeConfig, VisionLanguageConfig)
8
+from vllm.executor.executor_base import ExecutorAsyncBase
9
+from vllm.executor.gpu_executor import GPUExecutor
10
+from vllm.logger import init_logger
11
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
+from vllm.utils import make_async
13
+from vllm.worker.worker_base import WorkerWrapperBase
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+class XPUExecutor(GPUExecutor):
19
+
20
+ def __init__(
21
+ self,
22
+ model_config: ModelConfig,
23
+ cache_config: CacheConfig,
24
+ parallel_config: ParallelConfig,
25
+ scheduler_config: SchedulerConfig,
26
+ device_config: DeviceConfig,
27
+ load_config: LoadConfig,
28
+ lora_config: Optional[LoRAConfig],
29
+ vision_language_config: Optional[VisionLanguageConfig],
30
+ speculative_config: Optional[SpeculativeConfig],
31
+ ) -> None:
32
+ assert device_config.device_type == "xpu"
33
+ assert (not speculative_config
34
+ ), "Speculative decoding not yet supported for XPU backend"
35
+
36
+ model_config = _verify_and_get_model_config(model_config)
37
+
38
+ self.model_config = model_config
39
+ self.cache_config = cache_config
40
+ self.load_config = load_config
41
+ self.lora_config = lora_config
42
+ self.parallel_config = parallel_config
43
+ self.scheduler_config = scheduler_config
44
+ self.device_config = device_config
45
+ self.vision_language_config = vision_language_config
46
+ self.speculative_config = None
47
+
48
+ # Instantiate the worker and load the model to GPU.
49
+ self._init_executor()
50
+
51
+ def _create_worker(self,
52
+ local_rank: int = 0,
53
+ rank: int = 0,
54
+ distributed_init_method: Optional[str] = None):
55
+ if self.speculative_config is None:
56
+ worker_module_name = "vllm.worker.xpu_worker"
57
+ worker_class_name = "XPUWorker"
58
+ else:
59
+ raise NotImplementedError(
60
+ "XPU does not support speculative decoding")
61
+
62
+ wrapper = WorkerWrapperBase(
63
+ worker_module_name=worker_module_name,
64
+ worker_class_name=worker_class_name,
65
+ )
66
+ wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
67
+ distributed_init_method))
68
+ return wrapper.worker
69
+
70
+ def execute_model(
71
+ self,
72
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
73
+ output = self.driver_worker.execute_model(execute_model_req)
74
+ return output
75
+
76
+
77
+class XPUExecutorAsync(XPUExecutor, ExecutorAsyncBase):
78
+
79
+ async def execute_model_async(
80
+ self,
81
+ execute_model_req: ExecuteModelRequest,
82
+ ) -> List[SamplerOutput]:
83
+ output = await make_async(self.driver_worker.execute_model
84
+ )(execute_model_req=execute_model_req)
85
+ return output
86
+
87
+
88
+def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
89
+ if config.dtype == torch.bfloat16:
90
+ logger.warning(
91
+ "bfloat16 is not fully supported on XPU, casting to float16.")
92
+ config.dtype = torch.float16
93
+ if not config.enforce_eager:
94
+ logger.warning(
95
+ "CUDA graph is not supported on XPU, fallback to the eager "
96
+ "mode.")
97
+ config.enforce_eager = True
98
+ return config
vllm/model_executor/custom_op.py
+4
-4
@@ -1,6 +1,6 @@
1
import torch.nn as nn
2
3
-from vllm.utils import is_cpu, is_hip, is_tpu
3
+from vllm.utils import is_cpu, is_hip, is_tpu, is_xpu
4
5
6
class CustomOp(nn.Module):
@@ -29,9 +29,7 @@ class CustomOp(nn.Module):
29
return self.forward_cuda(*args, **kwargs)
30
31
def forward_xpu(self, *args, **kwargs):
32
- # By default, we assume that XPU ops are compatible with CUDA ops.
33
- # NOTE(woosuk): This is a placeholder for future extensions.
34
- return self.forward_cuda(*args, **kwargs)
32
+ raise NotImplementedError
33
34
def forward_cpu(self, *args, **kwargs):
35
# By default, we assume that CPU ops are compatible with CUDA ops.
@@ -58,5 +56,7 @@ class CustomOp(nn.Module):
56
return self.forward_cpu
57
elif is_tpu():
58
return self.forward_tpu
59
+ elif is_xpu():
60
+ return self.forward_xpu
61
else:
62
return self.forward_cuda
vllm/model_executor/layers/activation.py
+35
@@ -37,6 +37,15 @@ class SiluAndMul(CustomOp):
37
ops.silu_and_mul(out, x)
38
return out
39
40
+ def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
41
+ from vllm._ipex_ops import ipex_ops as ops
42
+
43
+ d = x.shape[-1] // 2
44
+ output_shape = (x.shape[:-1] + (d, ))
45
+ out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
46
+ ops.silu_and_mul(out, x)
47
+ return out
48
+
49
50
class GeluAndMul(CustomOp):
51
"""An activation function for GeGLU.
@@ -71,6 +80,18 @@ class GeluAndMul(CustomOp):
80
ops.gelu_tanh_and_mul(out, x)
81
return out
82
83
+ def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
84
+ from vllm._ipex_ops import ipex_ops as ops
85
+
86
+ d = x.shape[-1] // 2
87
+ output_shape = (x.shape[:-1] + (d, ))
88
+ out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
89
+ if self.approximate == "none":
90
+ ops.gelu_and_mul(out, x)
91
+ elif self.approximate == "tanh":
92
+ ops.gelu_tanh_and_mul(out, x)
93
+ return out
94
+
95
def extra_repr(self) -> str:
96
return f'approximate={repr(self.approximate)}'
97
@@ -90,6 +111,13 @@ class NewGELU(CustomOp):
111
ops.gelu_new(out, x)
112
return out
113
114
+ def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
115
+ from vllm._ipex_ops import ipex_ops as ops
116
+
117
+ out = torch.empty_like(x)
118
+ ops.gelu_new(out, x)
119
+ return out
120
+
121
122
class FastGELU(CustomOp):
123
@@ -105,6 +133,13 @@ class FastGELU(CustomOp):
133
ops.gelu_fast(out, x)
134
return out
135
136
+ def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
137
+ from vllm._ipex_ops import ipex_ops as ops
138
+
139
+ out = torch.empty_like(x)
140
+ ops.gelu_fast(out, x)
141
+ return out
142
+
143
144
class ScaledActivation(nn.Module):
145
"""An activation function with post-scale parameters.
vllm/model_executor/layers/layernorm.py
+24
@@ -67,6 +67,30 @@ class RMSNorm(CustomOp):
67
)
68
return out
69
70
+ def forward_xpu(
71
+ self,
72
+ x: torch.Tensor,
73
+ residual: Optional[torch.Tensor] = None,
74
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
75
+ from vllm._ipex_ops import ipex_ops as ops
76
+
77
+ if residual is not None:
78
+ ops.fused_add_rms_norm(
79
+ x,
80
+ residual,
81
+ self.weight.data,
82
+ self.variance_epsilon,
83
+ )
84
+ return x, residual
85
+ out = torch.empty_like(x)
86
+ ops.rms_norm(
87
+ out,
88
+ x,
89
+ self.weight.data,
90
+ self.variance_epsilon,
91
+ )
92
+ return out
93
+
94
def extra_repr(self) -> str:
95
s = f"hidden_size={self.weight.data.size(0)}"
96
s += f", eps={self.variance_epsilon}"
vllm/model_executor/layers/rotary_embedding.py
+23
@@ -221,6 +221,29 @@ class RotaryEmbedding(CustomOp):
221
self.cos_sin_cache, self.is_neox_style)
222
return query, key
223
224
+ def forward_xpu(
225
+ self,
226
+ positions: torch.Tensor,
227
+ query: torch.Tensor,
228
+ key: torch.Tensor,
229
+ offsets: Optional[torch.Tensor] = None,
230
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
231
+ from vllm._ipex_ops import ipex_ops as ops
232
+
233
+ self.cos_sin_cache = self.cos_sin_cache.to(positions.device,
234
+ dtype=query.dtype)
235
+ # ops.rotary_embedding()/batched_rotary_embedding()
236
+ # are in-place operations that update the query and key tensors.
237
+ if offsets is not None:
238
+ ops.batched_rotary_embedding(positions, query, key, self.head_size,
239
+ self.cos_sin_cache,
240
+ self.is_neox_style, self.rotary_dim,
241
+ offsets)
242
+ else:
243
+ ops.rotary_embedding(positions, query, key, self.head_size,
244
+ self.cos_sin_cache, self.is_neox_style)
245
+ return query, key
246
+
247
def forward_tpu(
248
self,
249
positions: torch.Tensor,
vllm/model_executor/layers/vocab_parallel_embedding.py
+1
-1
@@ -307,7 +307,7 @@ class VocabParallelEmbedding(torch.nn.Module):
307
else:
308
masked_input = input_
309
# Get the embeddings.
310
- output_parallel = F.embedding(masked_input, self.weight)
310
+ output_parallel = F.embedding(masked_input.long(), self.weight)
311
# Mask the output embedding.
312
if self.tp_size > 1:
313
output_parallel.masked_fill_(input_mask.unsqueeze(1), 0)
vllm/utils.py
+29
-2
@@ -160,6 +160,26 @@ def is_tpu() -> bool:
160
return libtpu is not None
161
162
163
+@lru_cache(maxsize=None)
164
+def is_xpu() -> bool:
165
+ from importlib.metadata import version
166
+ is_xpu_flag = "xpu" in version("vllm")
167
+ # vllm is not build with xpu
168
+ if not is_xpu_flag:
169
+ return False
170
+ try:
171
+ import intel_extension_for_pytorch as ipex # noqa: F401
172
+ _import_ipex = True
173
+ except ImportError as e:
174
+ logger.warning("Import Error for IPEX: %s", e.msg)
175
+ _import_ipex = False
176
+ # ipex dependency is not ready
177
+ if not _import_ipex:
178
+ logger.warning("not found ipex lib")
179
+ return False
180
+ return hasattr(torch, "xpu") and torch.xpu.is_available()
181
+
182
+
183
@lru_cache(maxsize=None)
184
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
185
"""Returns the maximum shared memory per thread block in bytes."""
@@ -482,6 +502,9 @@ def is_pin_memory_available() -> bool:
502
print_warning_once("Using 'pin_memory=False' as WSL is detected. "
503
"This may slow down the performance.")
504
return False
505
+ elif is_xpu():
506
+ print_warning_once("Pin memory is not supported on XPU.")
507
+ return False
508
elif is_neuron():
509
print_warning_once("Pin memory is not supported on Neuron.")
510
return False
@@ -497,8 +520,12 @@ class CudaMemoryProfiler:
520
521
def current_memory_usage(self) -> float:
522
# Return the memory usage in bytes.
500
- torch.cuda.reset_peak_memory_stats(self.device)
501
- mem = torch.cuda.max_memory_allocated(self.device)
523
+ if torch.cuda.is_available():
524
+ torch.cuda.reset_peak_memory_stats(self.device)
525
+ mem = torch.cuda.max_memory_allocated(self.device)
526
+ elif is_xpu():
527
+ torch.xpu.reset_peak_memory_stats(self.device)
528
+ mem = torch.xpu.max_memory_allocated(self.device)
529
return mem
530
531
def __enter__(self):
vllm/worker/cache_engine.py
+5
-2
@@ -4,7 +4,7 @@ from typing import List
4
import torch
5
6
from vllm.attention import get_attn_backend
7
-from vllm.config import CacheConfig, ModelConfig, ParallelConfig
7
+from vllm.config import CacheConfig, DeviceConfig, ModelConfig, ParallelConfig
8
from vllm.logger import init_logger
9
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size,
10
is_pin_memory_available)
@@ -25,10 +25,12 @@ class CacheEngine:
25
cache_config: CacheConfig,
26
model_config: ModelConfig,
27
parallel_config: ParallelConfig,
28
+ device_config: DeviceConfig,
29
) -> None:
30
self.cache_config = cache_config
31
self.model_config = model_config
32
self.parallel_config = parallel_config
33
+ self.device_config = device_config
34
35
self.head_size = model_config.get_head_size()
36
self.num_layers = model_config.get_num_layers(parallel_config)
@@ -55,7 +57,8 @@ class CacheEngine:
57
)
58
59
# Initialize the cache.
58
- self.gpu_cache = self._allocate_kv_cache(self.num_gpu_blocks, "cuda")
60
+ self.gpu_cache = self._allocate_kv_cache(
61
+ self.num_gpu_blocks, self.device_config.device_type)
62
self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks, "cpu")
63
64
def _allocate_kv_cache(
vllm/worker/worker.py
+2
-1
@@ -205,7 +205,8 @@ class Worker(WorkerBase):
205
def _init_cache_engine(self):
206
assert self.cache_config.num_gpu_blocks is not None
207
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
208
- self.parallel_config)
208
+ self.parallel_config,
209
+ self.device_config)
210
self.gpu_cache = self.cache_engine.gpu_cache
211
212
def _warm_up_model(self) -> None:
vllm/worker/xpu_model_runner.py
new
+417
@@ -0,0 +1,417 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import torch
4
+import torch.nn as nn
5
+
6
+from vllm.attention import get_attn_backend
7
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
8
+ ModelConfig, ParallelConfig, SchedulerConfig,
9
+ VisionLanguageConfig)
10
+from vllm.distributed import broadcast_tensor_dict
11
+from vllm.logger import init_logger
12
+from vllm.model_executor.model_loader import get_model
13
+from vllm.sampling_params import SamplingParams
14
+from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
+from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
16
+from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
17
+
18
+logger = init_logger(__name__)
19
+
20
+_PAD_SLOT_ID = -1
21
+_BATCH_SIZE_ALIGNMENT = 8
22
+_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
23
+ _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
24
+]
25
+
26
+
27
+class XPUModelRunner:
28
+
29
+ def __init__(
30
+ self,
31
+ model_config: ModelConfig,
32
+ parallel_config: ParallelConfig,
33
+ scheduler_config: SchedulerConfig,
34
+ device_config: DeviceConfig,
35
+ cache_config: CacheConfig,
36
+ load_config: LoadConfig,
37
+ lora_config: Optional[LoRAConfig],
38
+ vision_language_config: Optional[VisionLanguageConfig],
39
+ kv_cache_dtype: Optional[str] = "auto",
40
+ is_driver_worker: bool = False,
41
+ *args,
42
+ **kwargs,
43
+ ):
44
+ self.model_config = model_config
45
+ self.parallel_config = parallel_config
46
+ self.scheduler_config = scheduler_config
47
+ self.lora_config = lora_config
48
+ self.load_config = load_config
49
+ self.cache_config = cache_config
50
+ self.vision_language_config = vision_language_config
51
+ self.is_driver_worker = is_driver_worker
52
+
53
+ self.sliding_window = model_config.get_sliding_window()
54
+ self.device_config = device_config
55
+ self.device = self.device_config.device
56
+
57
+ self.kv_cache_dtype = kv_cache_dtype
58
+ self.block_size = cache_config.block_size
59
+ self.max_context_len_to_capture = (
60
+ self.model_config.max_context_len_to_capture
61
+ if self.model_config is not None else 0)
62
+
63
+ self.attn_backend = get_attn_backend(
64
+ self.model_config.get_num_attention_heads(self.parallel_config),
65
+ self.model_config.get_head_size(),
66
+ self.model_config.get_num_kv_heads(self.parallel_config),
67
+ self.model_config.get_sliding_window(),
68
+ self.model_config.dtype,
69
+ self.kv_cache_dtype,
70
+ self.block_size,
71
+ )
72
+
73
+ # Lazy initialization.
74
+ self.model: nn.Module # Set after init_Model
75
+
76
+ def load_model(self) -> None:
77
+ with CudaMemoryProfiler() as m:
78
+ self.model = get_model(
79
+ model_config=self.model_config,
80
+ device_config=self.device_config,
81
+ load_config=self.load_config,
82
+ lora_config=self.lora_config,
83
+ vision_language_config=self.vision_language_config,
84
+ parallel_config=self.parallel_config,
85
+ scheduler_config=self.scheduler_config,
86
+ cache_config=self.cache_config,
87
+ )
88
+
89
+ self.model_memory_usage = m.consumed_memory
90
+ logger.info("Loading model weights took %.4f GB",
91
+ self.model_memory_usage / float(2**30))
92
+
93
+ @property
94
+ def vocab_size(self) -> int:
95
+ return self.model_config.get_vocab_size()
96
+
97
+ @torch.inference_mode()
98
+ def profile_run(self) -> None:
99
+ # Enable top-k sampling to reflect the accurate memory usage.
100
+ sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
101
+ max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
102
+ max_num_seqs = self.scheduler_config.max_num_seqs
103
+
104
+ # Profile memory usage with max_num_sequences sequences and the total
105
+ # number of tokens equal to max_num_batched_tokens.
106
+ seqs: List[SequenceGroupMetadata] = []
107
+ # Additional GPU memory may be needed for vision encoding, which needs
108
+ # to be accounted for when calculating the GPU blocks for
109
+ # vLLM blocker manager.
110
+ # To exercise the worst scenario for GPU memory consumption,
111
+ # the number of seqs (batch_size) is chosen to maximize the number
112
+ # of images processed.
113
+ for group_id in range(max_num_seqs):
114
+ seq_len = (max_num_batched_tokens // max_num_seqs +
115
+ (group_id < max_num_batched_tokens % max_num_seqs))
116
+
117
+ seq_data = SequenceData([0] * seq_len)
118
+ dummy_multi_modal_data = None
119
+ seq = SequenceGroupMetadata(
120
+ request_id=str(group_id),
121
+ is_prompt=True,
122
+ seq_data={group_id: seq_data},
123
+ sampling_params=sampling_params,
124
+ block_tables=None,
125
+ lora_request=None,
126
+ multi_modal_data=dummy_multi_modal_data,
127
+ )
128
+ seqs.append(seq)
129
+
130
+ # Run the model with the dummy inputs.
131
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
132
+ kv_caches = [None] * num_layers
133
+ self.execute_model(seqs, kv_caches)
134
+ torch.xpu.synchronize()
135
+ return
136
+
137
+ def prepare_input_tensors(
138
+ self,
139
+ seq_group_metadata_list: List[SequenceGroupMetadata],
140
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
141
+ Optional[torch.Tensor]]:
142
+ multi_modal_input = None
143
+ if self.is_driver_worker:
144
+ # NOTE: We assume that all sequences in the group are all prompts or
145
+ # all decodes.
146
+ is_prompt = seq_group_metadata_list[0].is_prompt
147
+ # Prepare input tensors.
148
+ if is_prompt:
149
+ (input_tokens, input_positions, attn_metadata, seq_lens,
150
+ multi_modal_input
151
+ ) = self._prepare_prompt(seq_group_metadata_list)
152
+ else:
153
+ (input_tokens, input_positions,
154
+ attn_metadata) = self._prepare_decode(seq_group_metadata_list)
155
+ seq_lens = []
156
+ sampling_metadata = SamplingMetadata.prepare(
157
+ seq_group_metadata_list,
158
+ seq_lens,
159
+ # subquery_lens is not needed if chunked prefill is not
160
+ # supported. Since CPU worker doesn't support chunked prefill
161
+ # just use seq_lens instead.
162
+ seq_lens,
163
+ self.device,
164
+ pin_memory=False)
165
+ # Broadcast the metadata.
166
+ metadata_dict = {
167
+ "input_tokens": input_tokens,
168
+ "input_positions": input_positions,
169
+ "selected_token_indices":
170
+ sampling_metadata.selected_token_indices,
171
+ }
172
+ metadata_dict.update(attn_metadata.asdict_zerocopy())
173
+ broadcast_tensor_dict(metadata_dict, src=0)
174
+ else:
175
+ metadata_dict = broadcast_tensor_dict(src=0)
176
+ input_tokens = metadata_dict.pop("input_tokens")
177
+ input_positions = metadata_dict.pop("input_positions")
178
+ selected_token_indices = metadata_dict.pop(
179
+ "selected_token_indices")
180
+ attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
181
+ sampling_metadata = SamplingMetadata(
182
+ seq_groups=None,
183
+ selected_token_indices=selected_token_indices,
184
+ categorized_sample_indices=None,
185
+ num_prompts=0,
186
+ )
187
+
188
+ return (input_tokens, input_positions, attn_metadata,
189
+ sampling_metadata, multi_modal_input)
190
+
191
+ def _prepare_decode(
192
+ self,
193
+ seq_group_metadata_list: List[SequenceGroupMetadata],
194
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
195
+ assert len(seq_group_metadata_list) > 0
196
+ input_tokens: List[int] = []
197
+ input_positions: List[int] = []
198
+ slot_mapping: List[int] = []
199
+ seq_lens: List[int] = []
200
+ block_tables: List[List[int]] = []
201
+
202
+ for seq_group_metadata in seq_group_metadata_list:
203
+ assert not seq_group_metadata.is_prompt
204
+ assert seq_group_metadata.token_chunk_size == 1
205
+
206
+ seq_ids = list(seq_group_metadata.seq_data.keys())
207
+
208
+ for seq_id in seq_ids:
209
+ seq_data = seq_group_metadata.seq_data[seq_id]
210
+ generation_token = seq_data.get_last_token_id()
211
+ input_tokens.append(generation_token)
212
+
213
+ seq_len = seq_data.get_len()
214
+ position = seq_len - 1
215
+ input_positions.append(position)
216
+
217
+ seq_len = seq_len if self.sliding_window is None else min(
218
+ seq_len, self.sliding_window)
219
+ seq_lens.append(seq_len)
220
+
221
+ block_table = seq_group_metadata.block_tables[seq_id]
222
+ block_number = block_table[position // self.block_size]
223
+ block_offset = position % self.block_size
224
+ slot = block_number * self.block_size + block_offset
225
+ slot_mapping.append(slot)
226
+
227
+ if self.sliding_window is not None:
228
+ sliding_window_blocks = (self.sliding_window //
229
+ self.block_size)
230
+ block_table = block_table[-sliding_window_blocks:]
231
+ block_tables.append(block_table)
232
+
233
+ max_decode_seq_len = max(seq_lens)
234
+
235
+ input_tokens = torch.tensor(input_tokens,
236
+ dtype=torch.long,
237
+ device=self.device)
238
+ input_positions = torch.tensor(input_positions,
239
+ dtype=torch.long,
240
+ device=self.device)
241
+ slot_mapping = torch.tensor(slot_mapping,
242
+ dtype=torch.long,
243
+ device=self.device)
244
+ seq_lens_tensor = torch.tensor(seq_lens,
245
+ dtype=torch.int,
246
+ device=self.device)
247
+
248
+ max_block_table_len = max(
249
+ len(block_table) for block_table in block_tables)
250
+ block_tables = make_tensor_with_pad(
251
+ block_tables,
252
+ max_len=max_block_table_len,
253
+ pad=0,
254
+ dtype=torch.int,
255
+ device=self.device,
256
+ )
257
+
258
+ attn_metadata = self.attn_backend.make_metadata(
259
+ is_prompt=False,
260
+ slot_mapping=slot_mapping,
261
+ seq_lens=seq_lens,
262
+ seqlen_q=None,
263
+ max_seqlen=None,
264
+ seq_lens_tensor=seq_lens_tensor,
265
+ max_decode_seq_len=max_decode_seq_len,
266
+ num_prefill_tokens=0,
267
+ num_decode_tokens=len(input_tokens),
268
+ num_prefills=0,
269
+ block_tables=block_tables,
270
+ )
271
+ return (
272
+ input_tokens,
273
+ input_positions,
274
+ attn_metadata,
275
+ )
276
+
277
+ @torch.inference_mode()
278
+ def execute_model(
279
+ self,
280
+ seq_group_metadata_list: List[SequenceGroupMetadata],
281
+ kv_caches: List[torch.Tensor],
282
+ ) -> Optional[SamplerOutput]:
283
+ (input_tokens, input_positions, attn_metadata, sampling_metadata,
284
+ multi_modal_input
285
+ ) = self.prepare_input_tensors(seq_group_metadata_list)
286
+
287
+ model_executable = self.model
288
+ execute_model_kwargs = {
289
+ "input_ids": input_tokens,
290
+ "positions": input_positions,
291
+ "kv_caches": kv_caches,
292
+ "attn_metadata": attn_metadata,
293
+ }
294
+ if self.vision_language_config:
295
+ execute_model_kwargs.update({"image_input": multi_modal_input})
296
+
297
+ hidden_states = model_executable(**execute_model_kwargs)
298
+
299
+ # Compute the logits.
300
+ logits = self.model.compute_logits(hidden_states, sampling_metadata)
301
+
302
+ # Only perform sampling in the driver worker.
303
+ if not self.is_driver_worker:
304
+ return None
305
+
306
+ # Sample the next token.
307
+ output = self.model.sample(
308
+ logits=logits,
309
+ sampling_metadata=sampling_metadata,
310
+ )
311
+ return output
312
+
313
+ def _prepare_prompt(
314
+ self,
315
+ seq_group_metadata_list: List[SequenceGroupMetadata],
316
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
317
+ Optional[torch.Tensor]]:
318
+ assert len(seq_group_metadata_list) > 0
319
+ input_tokens: List[int] = []
320
+ input_positions: List[int] = []
321
+ slot_mapping: List[int] = []
322
+ seq_lens: List[int] = []
323
+ multi_modal_input_list: List[torch.Tensor] = []
324
+
325
+ for seq_group_metadata in seq_group_metadata_list:
326
+ assert seq_group_metadata.is_prompt
327
+ seq_ids = list(seq_group_metadata.seq_data.keys())
328
+ assert len(seq_ids) == 1
329
+ seq_id = seq_ids[0]
330
+
331
+ seq_data = seq_group_metadata.seq_data[seq_id]
332
+ prompt_tokens = seq_data.get_token_ids()
333
+ computed_len = seq_data.get_num_computed_tokens()
334
+ seq_len = len(prompt_tokens)
335
+
336
+ seq_lens.append(seq_len) # Prompt token num
337
+ input_tokens.extend(prompt_tokens) # Token ids
338
+
339
+ # Token position ids
340
+ # NOTE(woosuk): Here we assume that the first token in the prompt
341
+ # is always the first token in the sequence.
342
+ input_positions.extend(list(range(computed_len, seq_len)))
343
+
344
+ if seq_group_metadata.multi_modal_data:
345
+ multi_modal_input_list.append(
346
+ seq_group_metadata.multi_modal_data.data)
347
+
348
+ if seq_group_metadata.block_tables is None:
349
+ # During memory profiling, the block tables are not initialized
350
+ # yet. In this case, we just use a dummy slot mapping.
351
+ slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
352
+ continue
353
+
354
+ # Compute the slot mapping.
355
+ block_table = seq_group_metadata.block_tables[seq_id]
356
+ # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
357
+ # where start_idx is max(0, seq_len - sliding_window).
358
+ # For example, if the prompt len is 10, sliding window is 8, and
359
+ # block size is 4, the first two tokens are masked and the slot
360
+ # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
361
+ start_idx = 0
362
+ if self.sliding_window is not None:
363
+ start_idx = max(0, seq_len - self.sliding_window)
364
+
365
+ for i in range(computed_len, seq_len):
366
+ if i < start_idx:
367
+ slot_mapping.append(_PAD_SLOT_ID)
368
+ continue
369
+
370
+ block_number = block_table[i //
371
+ self.block_size] # type: ignore
372
+ block_offset = i % self.block_size # type: ignore
373
+ slot = block_number * self.block_size + block_offset
374
+ slot_mapping.append(slot)
375
+
376
+ if multi_modal_input_list:
377
+ assert self.vision_language_config, (
378
+ "Multi-modal inputs are only supported by "
379
+ "vision language models.")
380
+ multi_modal_input = torch.cat(multi_modal_input_list,
381
+ dim=0).to(self.device)
382
+ else:
383
+ multi_modal_input = None
384
+
385
+ num_prompt_tokens = len(input_tokens)
386
+
387
+ input_tokens = torch.tensor(input_tokens,
388
+ dtype=torch.long,
389
+ device=self.device) # type: ignore
390
+ input_positions = torch.tensor(input_positions,
391
+ dtype=torch.long,
392
+ device=self.device) # type: ignore
393
+ slot_mapping = torch.tensor(slot_mapping,
394
+ dtype=torch.long,
395
+ device=self.device) # type: ignore
396
+
397
+ max_seqlen = max(seq_lens)
398
+ tmp = [0]
399
+ tmp.extend(seq_lens)
400
+ seqlen = torch.tensor(tmp)
401
+ seqlen_q = torch.cumsum(seqlen, dim=0).to(device=self.device)
402
+
403
+ attn_metadata = self.attn_backend.make_metadata(
404
+ is_prompt=True,
405
+ slot_mapping=slot_mapping,
406
+ seq_lens=seq_lens,
407
+ seqlen_q=seqlen_q,
408
+ max_seqlen=max_seqlen,
409
+ seq_lens_tensor=None,
410
+ max_decode_seq_len=None,
411
+ num_prefills=len(seq_lens),
412
+ num_prefill_tokens=num_prompt_tokens,
413
+ num_decode_tokens=0,
414
+ block_tables=torch.tensor([], device=self.device, dtype=torch.int),
415
+ )
416
+ return (input_tokens, input_positions, attn_metadata, seq_lens,
417
+ multi_modal_input)
vllm/worker/xpu_worker.py
new
+193
@@ -0,0 +1,193 @@
1
+"""A XPU worker class."""
2
+import gc
3
+import os
4
+from typing import List, Optional, Tuple
5
+
6
+import intel_extension_for_pytorch # noqa: F401
7
+import oneccl_bindings_for_pytorch # noqa: F401
8
+import torch
9
+import torch.distributed
10
+
11
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
+ ModelConfig, ParallelConfig, SchedulerConfig,
13
+ SpeculativeConfig, VisionLanguageConfig)
14
+from vllm.distributed import (ensure_model_parallel_initialized,
15
+ init_distributed_environment)
16
+from vllm.logger import init_logger
17
+from vllm.model_executor import set_random_seed
18
+from vllm.utils import is_xpu
19
+from vllm.worker.cache_engine import CacheEngine
20
+from vllm.worker.worker import Worker
21
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase
22
+from vllm.worker.xpu_model_runner import XPUModelRunner
23
+
24
+logger = init_logger(__name__)
25
+
26
+
27
+class XPUWorker(LoraNotSupportedWorkerBase, Worker):
28
+ """A worker class that executes (a partition of) the model on a GPU.
29
+
30
+ Each worker is associated with a single XPU device. The worker is
31
+ responsible for maintaining the KV cache and executing the model on the
32
+ XPU. In case of distributed inference, each worker is assigned a partition
33
+ of the model.
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ model_config: ModelConfig,
39
+ parallel_config: ParallelConfig,
40
+ scheduler_config: SchedulerConfig,
41
+ device_config: DeviceConfig,
42
+ cache_config: CacheConfig,
43
+ load_config: LoadConfig,
44
+ local_rank: int,
45
+ rank: int,
46
+ distributed_init_method: str,
47
+ lora_config: Optional[LoRAConfig] = None,
48
+ vision_language_config: Optional[VisionLanguageConfig] = None,
49
+ speculative_config: Optional[SpeculativeConfig] = None,
50
+ is_driver_worker: bool = False,
51
+ ) -> None:
52
+ assert device_config.device_type == "xpu"
53
+ assert is_xpu()
54
+
55
+ self.model_config = model_config
56
+ self.parallel_config = parallel_config
57
+ self.scheduler_config = scheduler_config
58
+ self.device_config = device_config
59
+ self.cache_config = cache_config
60
+ self.load_config = load_config
61
+ self.local_rank = local_rank
62
+ self.rank = rank
63
+ self.distributed_init_method = distributed_init_method
64
+ self.lora_config = lora_config
65
+ self.is_driver_worker = is_driver_worker
66
+ if self.is_driver_worker:
67
+ assert self.rank == 0, "The driver worker must have rank 0."
68
+
69
+ self.vision_language_config = vision_language_config
70
+ if self.vision_language_config:
71
+ assert not self.lora_config, (
72
+ "To be tested: vision language model with LoRA settings.")
73
+
74
+ self.model_runner = XPUModelRunner( # type: ignore
75
+ model_config,
76
+ parallel_config,
77
+ scheduler_config,
78
+ device_config,
79
+ cache_config,
80
+ load_config=self.load_config,
81
+ lora_config=self.lora_config,
82
+ kv_cache_dtype=self.cache_config.cache_dtype,
83
+ is_driver_worker=is_driver_worker,
84
+ vision_language_config=vision_language_config,
85
+ )
86
+ # Uninitialized cache engine. Will be initialized by
87
+ # initialize_cache.
88
+ self.cache_engine: CacheEngine
89
+ self.gpu_cache: List[torch.Tensor]
90
+
91
+ def init_device(self) -> None:
92
+ if self.device_config.device.type == "xpu" and is_xpu():
93
+ self.device = torch.device(f"xpu:{self.local_rank}")
94
+ torch.xpu.set_device(self.device)
95
+ torch.xpu.empty_cache()
96
+ self.init_gpu_memory = torch.xpu.get_device_properties(
97
+ self.local_rank).total_memory
98
+ else:
99
+ raise RuntimeError(
100
+ f"Not support device type: {self.device_config.device}")
101
+ # Initialize the distributed environment.
102
+ self.init_worker_distributed_environment()
103
+ # Initialize the model.
104
+ set_random_seed(self.model_config.seed)
105
+
106
+ # keep this method for `empty_cache` and `synchronize` api
107
+ @torch.inference_mode()
108
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
109
+ """Profiles the peak memory usage of the model to determine how many
110
+ KV blocks may be allocated without OOMs.
111
+
112
+ The engine will first conduct a profiling of the existing memory usage.
113
+ Then, it calculate the maximum possible number of GPU and CPU blocks
114
+ that can be allocated with the remaining free memory.
115
+
116
+ .. tip::
117
+ You may limit the usage of GPU memory
118
+ by adjusting the `gpu_memory_utilization` parameter.
119
+ """
120
+ # Profile the memory usage of the model and get the maximum number of
121
+ # cache blocks that can be allocated with the remaining free memory.
122
+ torch.xpu.empty_cache()
123
+
124
+ # Execute a forward pass with dummy inputs to profile the memory usage
125
+ # of the model.
126
+ self.model_runner.profile_run()
127
+
128
+ # Calculate the number of blocks that can be allocated with the
129
+ # profiled peak memory.
130
+ torch.xpu.synchronize()
131
+ used_memory = torch.xpu.memory_allocated()
132
+ total_gpu_memory = torch.xpu.get_device_properties(
133
+ self.local_rank).total_memory
134
+ free_gpu_memory = total_gpu_memory - used_memory
135
+
136
+ # NOTE(woosuk): Here we assume that the other processes using the same
137
+ # GPU did not change their memory usage during the profiling.
138
+ peak_memory = self.init_gpu_memory - free_gpu_memory
139
+ assert peak_memory > 0, (
140
+ "Error in memory profiling. This happens when the GPU memory was "
141
+ "not properly cleaned up before initializing the vLLM instance.")
142
+
143
+ cache_block_size = self.get_cache_block_size_bytes()
144
+ num_gpu_blocks = int(
145
+ (total_gpu_memory * self.cache_config.gpu_memory_utilization -
146
+ peak_memory) // cache_block_size)
147
+ num_cpu_blocks = int(self.cache_config.swap_space_bytes //
148
+ cache_block_size)
149
+ num_gpu_blocks = max(num_gpu_blocks, 0)
150
+ num_cpu_blocks = max(num_cpu_blocks, 0)
151
+ gc.collect()
152
+ torch.xpu.empty_cache()
153
+ return num_gpu_blocks, num_cpu_blocks
154
+
155
+ def _warm_up_model(self) -> None:
156
+ # IPEX don't support capture graph yet
157
+ pass
158
+
159
+ def init_worker_distributed_environment(self) -> None:
160
+ """Initialize the distributed environment."""
161
+
162
+ parallel_config = self.parallel_config
163
+ rank = self.rank
164
+ distributed_init_method = self.distributed_init_method
165
+
166
+ if torch.distributed.is_initialized():
167
+ torch_world_size = torch.distributed.get_world_size()
168
+ if torch_world_size != parallel_config.world_size:
169
+ raise RuntimeError(
170
+ "torch.distributed is already initialized but the torch "
171
+ "world size does not match parallel_config.world_size "
172
+ f"({torch_world_size} vs. {parallel_config.world_size}).")
173
+ elif not distributed_init_method:
174
+ raise ValueError(
175
+ "distributed_init_method must be set if torch.distributed "
176
+ "is not already initialized")
177
+ else:
178
+ # use sockets as default Level zero IPC exchange backend. By
179
+ # default oneccl will use `drmfd` as mechanism which need extra
180
+ # dependency (libdrm and drm headers) on your system.
181
+ ENV_CCL_ZE_IPC_EXCHANGE = os.getenv("CCL_ZE_IPC_EXCHANGE",
182
+ "sockets")
183
+ os.environ['CCL_ZE_IPC_EXCHANGE'] = ENV_CCL_ZE_IPC_EXCHANGE
184
+ init_distributed_environment(
185
+ world_size=parallel_config.world_size,
186
+ rank=rank,
187
+ distributed_init_method=distributed_init_method,
188
+ local_rank=self.local_rank,
189
+ backend="ccl")
190
+
191
+ ensure_model_parallel_initialized(
192
+ parallel_config.tensor_parallel_size,
193
+ parallel_config.pipeline_parallel_size)