@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b69781f107
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 04:45 UTC

[Hardware][Intel GPU] Add v1 Intel GPU support with Flash attention backend. (#19560)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>

Kunshang Ji committed Jun 27, 2025 at 00:27 UTC b69781f107b7ad847a351f584178cfafbee2b32a
10 files changed +394 -43
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+1
@@ -28,4 +28,5 @@ docker run \
28 sh -c '
29 VLLM_USE_V1=0 python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m
30 VLLM_USE_V1=0 python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m -tp 2
31 + VLLM_USE_V1=1 python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
32 '
docker/Dockerfile.xpu
+1
@@ -35,6 +35,7 @@ RUN --mount=type=bind,source=.git,target=.git \
35 if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
36
37 ENV VLLM_TARGET_DEVICE=xpu
38 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
39
40 RUN --mount=type=cache,target=/root/.cache/pip \
41 --mount=type=bind,source=.git,target=.git \
requirements/xpu.txt
+1
@@ -9,6 +9,7 @@ setuptools>=77.0.3,<80.0.0
9 wheel
10 jinja2>=3.1.6
11 datasets # for benchmark scripts
12 +numba == 0.60.0 # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
13
14 torch==2.7.0+xpu
15 torchaudio
vllm/_ipex_ops.py
+105
@@ -228,6 +228,111 @@ class ipex_ops:
228 ipex.llm.modules.PagedAttention.reshape_and_cache(
229 key, value, key_cache, value_cache, slot_mapping)
230
231 + @staticmethod
232 + def reshape_and_cache_flash(
233 + key: torch.Tensor,
234 + value: torch.Tensor,
235 + key_cache: torch.Tensor,
236 + value_cache: torch.Tensor,
237 + slot_mapping: torch.Tensor,
238 + kv_cache_dtype: str,
239 + k_scale: Optional[torch.Tensor] = None,
240 + v_scale: Optional[torch.Tensor] = None,
241 + k_scale_float: float = 1.0,
242 + v_scale_float: float = 1.0,
243 + ) -> None:
244 + assert kv_cache_dtype == "auto"
245 + # TODO: support FP8 kv cache.
246 + ipex.llm.modules.PagedAttention.reshape_and_cache_flash(
247 + key, value, key_cache, value_cache, slot_mapping)
248 +
249 + @staticmethod
250 + def flash_attn_varlen_func(
251 + out: torch.Tensor,
252 + q: torch.Tensor,
253 + k: torch.Tensor,
254 + v: torch.Tensor,
255 + cu_seqlens_q: torch.Tensor,
256 + seqused_k: torch.Tensor, # we don't support this in ipex kernel
257 + max_seqlen_q: int,
258 + max_seqlen_k: int,
259 + softmax_scale: float,
260 + causal: bool,
261 + block_table: torch.Tensor,
262 + alibi_slopes: Optional[torch.Tensor],
263 + window_size: Optional[list[int]] = None,
264 + softcap: Optional[float] = 0.0,
265 + cu_seqlens_k: Optional[torch.Tensor] = None,
266 + # The following parameters are not used in ipex kernel currently,
267 + # we keep API compatible to CUDA's.
268 + scheduler_metadata=None,
269 + fa_version: int = 2,
270 + q_descale=None,
271 + k_descale=None,
272 + v_descale=None,
273 + ):
274 + if cu_seqlens_k is None:
275 + # cu_seqlens_k is not used in ipex kernel.
276 + cu_seqlens_k = torch.cumsum(seqused_k, dim=0)
277 + cu_seqlens_k = torch.cat([
278 + torch.tensor([0], device=seqused_k.device, dtype=torch.int32),
279 + cu_seqlens_k
280 + ]).to(torch.int32)
281 +
282 + real_window_size: tuple[int, int]
283 + if window_size is None:
284 + real_window_size = (-1, -1)
285 + else:
286 + assert len(window_size) == 2
287 + real_window_size = (window_size[0], window_size[1])
288 + return ipex.llm.modules.PagedAttention.flash_attn_varlen_func(
289 + out,
290 + q.contiguous(),
291 + k,
292 + v,
293 + cu_seqlens_q,
294 + cu_seqlens_k,
295 + max_seqlen_q,
296 + max_seqlen_k,
297 + softmax_scale,
298 + causal,
299 + block_table,
300 + alibi_slopes,
301 + softcap=softcap,
302 + window_size_left=real_window_size[0],
303 + window_size_right=real_window_size[1],
304 + k_scale=1.0,
305 + v_scale=1.0,
306 + )
307 +
308 + @staticmethod
309 + def get_scheduler_metadata(
310 + batch_size,
311 + max_seqlen_q,
312 + max_seqlen_k,
313 + num_heads_q,
314 + num_heads_kv,
315 + headdim,
316 + cache_seqlens: torch.Tensor,
317 + qkv_dtype=torch.bfloat16,
318 + headdim_v=None,
319 + cu_seqlens_q: Optional[torch.Tensor] = None,
320 + cu_seqlens_k_new: Optional[torch.Tensor] = None,
321 + cache_leftpad: Optional[torch.Tensor] = None,
322 + page_size: Optional[int] = None,
323 + max_seqlen_k_new=0,
324 + causal=False,
325 + window_size=(-1, -1), # -1 means infinite context window
326 + has_softcap=False,
327 + num_splits=0, # Can be tuned for speed
328 + pack_gqa=None, # Can be tuned for speed
329 + sm_margin=0, # Can be tuned if some SMs are used for communication
330 + ) -> None:
331 + logger.warning_once(
332 + "get_scheduler_metadata is not implemented for ipex_ops, "
333 + "returning None.")
334 + return None
335 +
336 @staticmethod
337 def copy_blocks(key_caches: list[torch.Tensor],
338 value_caches: list[torch.Tensor],
vllm/attention/utils/fa_utils.py
+14 -1
@@ -4,13 +4,27 @@ from typing import Optional
4
5 from vllm import envs
6 from vllm.logger import init_logger
7 +from vllm.platforms import current_platform
8
9 logger = init_logger(__name__)
10
11 +if current_platform.is_cuda():
12 + from vllm import _custom_ops as ops
13 + reshape_and_cache_flash = ops.reshape_and_cache_flash
14 + from vllm.vllm_flash_attn import (flash_attn_varlen_func,
15 + get_scheduler_metadata)
16 +elif current_platform.is_xpu():
17 + from vllm._ipex_ops import ipex_ops as ops
18 + reshape_and_cache_flash = ops.reshape_and_cache_flash
19 + flash_attn_varlen_func = ops.flash_attn_varlen_func
20 + get_scheduler_metadata = ops.get_scheduler_metadata
21 +
22
23 def get_flash_attn_version(requires_alibi: bool = False) -> Optional[int]:
24 # import here to avoid circular dependencies
25 from vllm.platforms import current_platform
26 + if current_platform.is_xpu():
27 + return 2
28 try:
29 from vllm.vllm_flash_attn.flash_attn_interface import (
30 fa_version_unsupported_reason, is_fa_version_supported)
@@ -50,6 +64,5 @@ def get_flash_attn_version(requires_alibi: bool = False) -> Optional[int]:
64
65
66 def flash_attn_supports_fp8() -> bool:
53 - from vllm.platforms import current_platform
67 return get_flash_attn_version() == 3 and \
68 current_platform.get_device_capability().major == 9
vllm/executor/ray_distributed_executor.py
+1 -1
@@ -73,7 +73,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
73
74 def _init_executor(self) -> None:
75 self.forward_dag: Optional[ray.dag.CompiledDAG] = None
76 - if envs.VLLM_USE_V1:
76 + if envs.VLLM_USE_V1 and not current_platform.is_xpu():
77 # V1 uses SPMD worker and compiled DAG
78 os.environ["VLLM_USE_RAY_SPMD_WORKER"] = "1"
79 os.environ["VLLM_USE_RAY_COMPILED_DAG"] = "1"
vllm/platforms/xpu.py
+70 -34
@@ -1,18 +1,21 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +import os
5 from typing import TYPE_CHECKING, Optional
6
7 import torch
8
9 +import vllm.envs as envs
10 from vllm.logger import init_logger
11 from vllm.utils import DEFAULT_MAX_NUM_BATCHED_TOKENS
12
13 from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
14
15 if TYPE_CHECKING:
14 - from vllm.config import VllmConfig
16 + from vllm.config import ModelConfig, VllmConfig
17 else:
18 + ModelConfig = None
19 VllmConfig = None
20
21 logger = init_logger(__name__)
@@ -35,8 +38,13 @@ class XPUPlatform(Platform):
38 use_mla: bool) -> str:
39 if selected_backend != _Backend.IPEX:
40 logger.info("Cannot use %s backend on XPU.", selected_backend)
38 - logger.info("Using IPEX attention backend.")
39 - return "vllm.attention.backends.ipex_attn.IpexAttnBackend"
41 + use_v1 = envs.VLLM_USE_V1
42 + if use_v1:
43 + logger.info("Using Flash Attention backend on V1 engine.")
44 + return "vllm.v1.attention.backends.flash_attn.FlashAttentionBackend"
45 + else:
46 + logger.info("Using IPEX attention backend.")
47 + return "vllm.attention.backends.ipex_attn.IpexAttnBackend"
48
49 @classmethod
50 def get_device_capability(
@@ -67,25 +75,27 @@ class XPUPlatform(Platform):
75 @classmethod
76 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
77 cache_config = vllm_config.cache_config
78 + # in V1(or with ipex chunked prefill) block_size is 64
79 if cache_config and cache_config.block_size is None:
71 - cache_config.block_size = 16
72 -
73 - # check and update model config
74 - model_config = vllm_config.model_config
75 - if model_config.dtype == torch.bfloat16:
76 - bf16_supported = cls.device_support_bf16()
77 - if not bf16_supported:
80 + if envs.VLLM_USE_V1:
81 + cache_config.block_size = 64
82 + else:
83 + cache_config.block_size = 16
84 +
85 + # Instances created using VllmConfig() typically have model_config as
86 + # None by default. The modification involves adding a check to prevent
87 + # potential null exceptions check and update model config.
88 + if vllm_config.model_config is not None:
89 + model_config = vllm_config.model_config
90 + if model_config.dtype == torch.bfloat16:
91 + bf16_supported = cls.device_support_bf16()
92 + if not bf16_supported:
93 + model_config.dtype = torch.float16
94 + if not model_config.enforce_eager:
95 logger.warning(
79 - "bfloat16 is only supported on Intel Data Center GPU, "
80 - "Intel Arc GPU is not supported yet. Your device is %s,"
81 - " which is not supported. will fallback to float16",
82 - cls.get_device_name())
83 - model_config.dtype = torch.float16
84 - if not model_config.enforce_eager:
85 - logger.warning(
86 - "CUDA graph is not supported on XPU, fallback to the eager "
87 - "mode.")
88 - model_config.enforce_eager = True
96 + "CUDA graph is not supported on XPU, fallback to the eager "
97 + "mode.")
98 + model_config.enforce_eager = True
99
100 if vllm_config.speculative_config is not None:
101 raise NotImplementedError(
@@ -96,21 +106,27 @@ class XPUPlatform(Platform):
106
107 # check and update parallel config
108 parallel_config = vllm_config.parallel_config
99 - if parallel_config.worker_cls == "auto":
109 + if envs.VLLM_USE_V1:
110 + parallel_config.worker_cls =\
111 + "vllm.v1.worker.xpu_worker.XPUWorker"
112 + else:
113 parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
114
115 if parallel_config.distributed_executor_backend is None:
103 - parallel_config.distributed_executor_backend = "ray"
116 + if parallel_config.world_size > 1:
117 + parallel_config.distributed_executor_backend = "ray"
118 + else:
119 + parallel_config.distributed_executor_backend = "uni"
120 elif parallel_config.distributed_executor_backend == "mp":
121 # FIXME(kunshang):
122 # spawn needs calling `if __name__ == '__main__':``
123 # fork is not supported for xpu start new process.
108 - logger.error(
109 - "Both start methods (spawn and fork) have issue "
110 - "on XPU if you use mp backend, setting it to ray instead.")
111 - parallel_config.distributed_executor_backend = "ray"
112 -
113 - elif parallel_config.distributed_executor_backend != "ray":
124 + if envs.VLLM_WORKER_MULTIPROC_METHOD != "spawn":
125 + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
126 + logger.warning(
127 + "Please use spawn as start method if you want to use mp.")
128 + elif parallel_config.distributed_executor_backend != "ray" and \
129 + parallel_config.distributed_executor_backend != "uni":
130 logger.warning(
131 "%s is not supported on XPU, fallback to ray distributed"
132 " executor backend.",
@@ -142,15 +158,35 @@ class XPUPlatform(Platform):
158 @classmethod
159 def device_support_bf16(cls) -> bool:
160 device_name = cls.get_device_name().lower()
145 - if device_name.count("arc") > 0:
161 + if cls.is_client_gpu_a770():
162 + logger.warning("Intel Arc A770 have bfloat16 accuracy known issue,"
163 + " fallback to float16")
164 return False
147 - elif device_name.count("data center gpu") > 0:
148 - return True
165 else:
150 - logger.warning("Unknown device name %s, always use float16",
151 - device_name)
152 - return False
166 + logger.info(
167 + "Device name %s supports bfloat16. Please file an issue "
168 + "if you encounter any accuracy problems with bfloat16.",
169 + device_name)
170 + return True
171 +
172 + @classmethod
173 + def is_data_center_gpu(cls) -> bool:
174 + device_name = cls.get_device_name().lower()
175 + return device_name.count("data center gpu") > 0
176 +
177 + @classmethod
178 + def is_client_gpu_a770(cls) -> bool:
179 + device_name = cls.get_device_name().lower()
180 + return device_name.count("a770") > 0
181
182 @classmethod
183 def get_device_communicator_cls(cls) -> str:
184 return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator" # noqa
185 +
186 + @classmethod
187 + def supports_v1(cls, model_config: ModelConfig) -> bool:
188 + return True
189 +
190 + @classmethod
191 + def device_count(cls) -> int:
192 + return torch.xpu.device_count()
vllm/v1/attention/backends/flash_attn.py
+5 -7
@@ -14,10 +14,12 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 from vllm.attention.layer import Attention
15 from vllm.attention.ops.merge_attn_states import merge_attn_states
16 from vllm.attention.utils.fa_utils import (flash_attn_supports_fp8,
17 - get_flash_attn_version)
17 + flash_attn_varlen_func,
18 + get_flash_attn_version,
19 + get_scheduler_metadata,
20 + reshape_and_cache_flash)
21 from vllm.config import VllmConfig, get_layers_from_vllm_config
22 from vllm.logger import init_logger
20 -from vllm.platforms import current_platform
23 from vllm.utils import cdiv
24 from vllm.v1.attention.backends.utils import (
25 AttentionMetadataBuilder, CommonAttentionMetadata, get_kv_cache_layout,
@@ -28,10 +30,6 @@ from vllm.v1.worker.block_table import BlockTable
30 if TYPE_CHECKING:
31 from vllm.v1.worker.gpu_model_runner import GPUModelRunner
32
31 -if current_platform.is_cuda():
32 - from vllm.vllm_flash_attn import (flash_attn_varlen_func,
33 - get_scheduler_metadata)
34 -
33 logger = init_logger(__name__)
34
35
@@ -443,7 +441,7 @@ class FlashAttentionImpl(AttentionImpl):
441 # and value[:num_actual_tokens] because the reshape_and_cache_flash
442 # op uses the slot_mapping's shape to determine the number of
443 # actual tokens.
446 - torch.ops._C_cache_ops.reshape_and_cache_flash(
444 + reshape_and_cache_flash(
445 key,
446 value,
447 key_cache,
vllm/v1/worker/xpu_model_runner.py new
+32
@@ -0,0 +1,32 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +from typing import TYPE_CHECKING
3 +
4 +import torch
5 +
6 +from vllm.config import VllmConfig
7 +from vllm.logger import init_logger
8 +from vllm.v1.worker.gpu_model_runner import GPUModelRunner
9 +
10 +if TYPE_CHECKING:
11 + pass
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +class XPUModelRunner(GPUModelRunner):
17 + """A model runner for XPU devices."""
18 +
19 + def __init__(
20 + self,
21 + vllm_config: VllmConfig,
22 + device: torch.device,
23 + ):
24 + super().__init__(vllm_config, device)
25 + # FIXME: To be verified.
26 + self.cascade_attn_enabled = False
27 +
28 + def _init_device_properties(self) -> None:
29 + pass
30 +
31 + def _sync_device(self) -> None:
32 + torch.xpu.synchronize()
vllm/v1/worker/xpu_worker.py new
+164
@@ -0,0 +1,164 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import os
3 +
4 +import torch
5 +import torch.distributed
6 +
7 +import vllm.envs as envs
8 +from vllm.config import VllmConfig
9 +from vllm.logger import init_logger
10 +from vllm.model_executor import set_random_seed
11 +from vllm.platforms import current_platform
12 +from vllm.v1.worker.gpu_worker import (Worker,
13 + init_worker_distributed_environment)
14 +from vllm.v1.worker.xpu_model_runner import XPUModelRunner
15 +
16 +logger = init_logger(__name__)
17 +
18 +
19 +class XPUWorker(Worker):
20 + """A XPU worker class."""
21 +
22 + def __init__(
23 + self,
24 + vllm_config: VllmConfig,
25 + local_rank: int,
26 + rank: int,
27 + distributed_init_method: str,
28 + is_driver_worker: bool = False,
29 + ):
30 + super().__init__(vllm_config, local_rank, rank,
31 + distributed_init_method, is_driver_worker)
32 + device_config = self.device_config
33 + assert device_config.device_type == "xpu"
34 + assert current_platform.is_xpu()
35 +
36 + # Torch profiler. Enabled and configured through env vars:
37 + # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
38 + if envs.VLLM_TORCH_PROFILER_DIR:
39 + torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR
40 + logger.info("Profiling enabled. Traces will be saved to: %s",
41 + torch_profiler_trace_dir)
42 + self.profiler = torch.profiler.profile(
43 + activities=[
44 + torch.profiler.ProfilerActivity.CPU,
45 + torch.profiler.ProfilerActivity.XPU,
46 + ],
47 + with_stack=True,
48 + on_trace_ready=torch.profiler.tensorboard_trace_handler(
49 + torch_profiler_trace_dir, use_gzip=True))
50 + else:
51 + self.profiler = None
52 +
53 + # we provide this function due to `torch.xpu.mem_get_info()` doesn't
54 + # return correct free_gpu_memory on intel client GPU. We need to
55 + # calculate/estiamte it.
56 + def xpu_get_mem_info(self):
57 + if current_platform.is_data_center_gpu():
58 + return torch.xpu.mem_get_info()
59 + else:
60 + _, total_gpu_memory = torch.xpu.mem_get_info()
61 + # FIXME: memory_allocated() doesn't count non-torch allocations,
62 + # and we don't have any API to get it. so we mark it as 128MB.
63 + used_memory = torch.xpu.memory_allocated()
64 + non_torch_allocations = 128 * 1024 * 1024
65 + free_gpu_memory = total_gpu_memory - (used_memory +
66 + non_torch_allocations)
67 + return free_gpu_memory, total_gpu_memory
68 +
69 + @torch.inference_mode()
70 + def determine_available_memory(self) -> int:
71 + """Profiles the peak memory usage of the model to determine how many
72 + KV blocks may be allocated without OOMs.
73 + The engine will first conduct a profiling of the existing memory usage.
74 + Then, it calculate the maximum possible number of GPU and CPU blocks
75 + that can be allocated with the remaining free memory.
76 + .. tip::
77 + You may limit the usage of GPU memory
78 + by adjusting the `gpu_memory_utilization` parameter.
79 + """
80 + # Profile the memory usage of the model and get the maximum number of
81 + # cache blocks that can be allocated with the remaining free memory.
82 + torch.xpu.empty_cache()
83 + torch.xpu.reset_peak_memory_stats()
84 +
85 + free_gpu_memory, total_gpu_memory = torch.xpu.mem_get_info()
86 + current_allocated_bytes = torch.xpu.memory_allocated()
87 + msg = ("Before memory profiling run, "
88 + f"total GPU memory: {total_gpu_memory / 1024**2:.2f} MB, "
89 + f"model load takes {current_allocated_bytes / 1024**2:.2f} MB, "
90 + f"free gpu memory is {free_gpu_memory / 1024**2:.2f} MB.")
91 + logger.info(msg)
92 + # Execute a forward pass with dummy inputs to profile the memory usage
93 + # of the model.
94 + self.model_runner.profile_run()
95 +
96 + free_gpu_memory, _ = self.xpu_get_mem_info()
97 + # NOTE(woosuk): Here we assume that the other processes using the same
98 + # GPU did not change their memory usage during the profiling.
99 + assert self.init_gpu_memory > free_gpu_memory, (
100 + "Error in memory profiling. "
101 + f"Initial free memory {self.init_gpu_memory}, current free memory"
102 + f" {free_gpu_memory}. This happens when the GPU memory was "
103 + "not properly cleaned up before initializing the vLLM instance.")
104 +
105 + # Get the peak memory allocation recorded by torch
106 + peak_memory = torch.xpu.memory_stats()["allocated_bytes.all.peak"]
107 +
108 + torch.xpu.empty_cache()
109 + torch_allocated_bytes = torch.xpu.memory_stats(
110 + )["allocated_bytes.all.current"]
111 + total_allocated_bytes = self.xpu_get_mem_info(
112 + )[1] - self.xpu_get_mem_info()[0]
113 +
114 + non_torch_allocations = total_allocated_bytes - torch_allocated_bytes
115 + if non_torch_allocations > 0:
116 + peak_memory += non_torch_allocations
117 + available_kv_cache_memory = (
118 + total_gpu_memory * self.cache_config.gpu_memory_utilization -
119 + peak_memory)
120 +
121 + msg = ("After memory profiling run, "
122 + f"peak memory usage is {peak_memory / 1024**2:.2f} MB,"
123 + f"torch mem is {torch_allocated_bytes / 1024**2:.2f} MB, "
124 + f"non-torch mem is {non_torch_allocations / 1024**2:.2f} MB, "
125 + f"free gpu memory is {free_gpu_memory / 1024**2:.2f} MB.")
126 + logger.info(msg)
127 +
128 + return int(available_kv_cache_memory)
129 +
130 + def init_device(self):
131 + if self.device_config.device.type == "xpu" and current_platform.is_xpu(
132 + ):
133 + self.device = torch.device(f"xpu:{self.local_rank}")
134 + torch.xpu.set_device(self.device)
135 + torch.xpu.empty_cache()
136 + self.init_gpu_memory = torch.xpu.get_device_properties(
137 + self.local_rank).total_memory
138 + else:
139 + raise RuntimeError(
140 + f"Not support device type: {self.device_config.device}")
141 +
142 + ENV_CCL_ZE_IPC_EXCHANGE = os.getenv("CCL_ZE_IPC_EXCHANGE", "drmfd")
143 + ENV_CCL_ATL_TRANSPORT = os.getenv("CCL_ATL_TRANSPORT", "ofi")
144 + ENV_LOCAL_WORLD_SIZE = os.getenv("LOCAL_WORLD_SIZE",
145 + str(self.parallel_config.world_size))
146 + os.environ["CCL_ZE_IPC_EXCHANGE"] = ENV_CCL_ZE_IPC_EXCHANGE
147 + os.environ["CCL_ATL_TRANSPORT"] = ENV_CCL_ATL_TRANSPORT
148 + os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE
149 + os.environ["LOCAL_RANK"] = str(self.local_rank)
150 + dist_backend = "ccl"
151 +
152 + init_worker_distributed_environment(self.vllm_config, self.rank,
153 + self.distributed_init_method,
154 + self.local_rank, dist_backend)
155 +
156 + # global all_reduce needed for overall oneccl warm up
157 + torch.distributed.all_reduce(torch.zeros(1).xpu())
158 +
159 + # Set random seed.
160 + set_random_seed(self.model_config.seed)
161 +
162 + # Construct the model runner
163 + self.model_runner = XPUModelRunner( # type: ignore
164 + self.vllm_config, self.device)