| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import contextlib |
| 5 | import os |
| 6 | from typing import TYPE_CHECKING |
| 7 | |
| 8 | import torch |
| 9 | |
| 10 | # import custom ops, trigger op registration |
| 11 | import vllm_xpu_kernels._C # noqa |
| 12 | import vllm_xpu_kernels._moe_C # noqa |
| 13 | import vllm_xpu_kernels._xpu_C # noqa |
| 14 | |
| 15 | import vllm.envs as envs |
| 16 | from vllm.logger import init_logger |
| 17 | from vllm.v1.attention.backends.registry import AttentionBackendEnum |
| 18 | |
| 19 | from .interface import DeviceCapability, Platform, PlatformEnum |
| 20 | |
| 21 | if TYPE_CHECKING: |
| 22 | from vllm.config import VllmConfig |
| 23 | from vllm.config.kernel import IrOpPriorityConfig |
| 24 | from vllm.v1.attention.selector import AttentionSelectorConfig |
| 25 | else: |
| 26 | VllmConfig = None |
| 27 | |
| 28 | logger = init_logger(__name__) |
| 29 | |
| 30 | |
| 31 | def get_mem_info_wrapper( |
| 32 | device: int | str | torch.device | None = None, |
| 33 | ) -> tuple[int, int]: |
| 34 | """Get memory info for a device, matching `torch.accelerator.get_memory_info`. |
| 35 | |
| 36 | Args: |
| 37 | device: Device specification. Can be: |
| 38 | - None: Use current XPU device |
| 39 | - int: Device index |
| 40 | - str: Device string (e.g., "xpu:0", "xpu") |
| 41 | - torch.device: Device object |
| 42 | |
| 43 | Returns: |
| 44 | Tuple[int, int]: (free_memory, total_memory) in bytes |
| 45 | |
| 46 | """ |
| 47 | # Handle None - use current device |
| 48 | if device is None: |
| 49 | device = torch.xpu.current_device() |
| 50 | |
| 51 | # Handle torch.device objects |
| 52 | elif isinstance(device, torch.device): |
| 53 | if device.type != "xpu": |
| 54 | raise RuntimeError(f"Expected 'xpu' device, got '{device.type}'") |
| 55 | # If device index is not specified, use current device |
| 56 | device = ( |
| 57 | device.index if device.index is not None else torch.xpu.current_device() |
| 58 | ) |
| 59 | |
| 60 | # Handle string device specifications (e.g., "xpu:0", "xpu") |
| 61 | elif isinstance(device, str): |
| 62 | if not device.startswith("xpu"): |
| 63 | raise RuntimeError(f"Expected 'xpu' device string, got '{device}'") |
| 64 | # Parse device string |
| 65 | parts = device.split(":") |
| 66 | if len(parts) == 1: |
| 67 | # "xpu" -> use current device |
| 68 | device = torch.xpu.current_device() |
| 69 | elif len(parts) == 2: |
| 70 | # "xpu:0" -> use index 0 |
| 71 | try: |
| 72 | device = int(parts[1]) |
| 73 | except ValueError as err: |
| 74 | raise RuntimeError( |
| 75 | f"Invalid device index: '{device}', expected integer after ':'" |
| 76 | ) from err |
| 77 | else: |
| 78 | raise RuntimeError(f"Invalid device string format: '{device}'") |
| 79 | |
| 80 | # At this point, device should be an int |
| 81 | if isinstance(device, int): |
| 82 | # bounds check |
| 83 | device_count = torch.xpu.device_count() |
| 84 | if not (0 <= device < device_count): |
| 85 | raise ValueError( |
| 86 | f"Invalid device index {device}, must be in range [0, {device_count})" |
| 87 | ) |
| 88 | |
| 89 | elif not isinstance(device, int): |
| 90 | raise TypeError( |
| 91 | f"device must be int, str, torch.device, or None, got {type(device)}" |
| 92 | ) |
| 93 | |
| 94 | # Call the underlying C++ implementation |
| 95 | free, total = torch.ops._C_cache_ops.getMemoryInfo(device) |
| 96 | |
| 97 | return free, total |
| 98 | |
| 99 | |
| 100 | torch.accelerator.get_memory_info = get_mem_info_wrapper |
| 101 | |
| 102 | |
| 103 | class XPUPlatform(Platform): |
| 104 | _enum = PlatformEnum.XPU |
| 105 | device_name: str = "xpu" |
| 106 | device_type: str = "xpu" |
| 107 | dispatch_key: str = "XPU" |
| 108 | # Intel XPU's device key is "GPU" for Ray. |
| 109 | # see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501 |
| 110 | ray_device_key: str = "GPU" |
| 111 | dist_backend: str = "xccl" # xccl only |
| 112 | device_control_env_var: str = "ZE_AFFINITY_MASK" |
| 113 | supported_quantization: list[str] = [ |
| 114 | "awq", |
| 115 | "gptq", |
| 116 | "moe_wna16", |
| 117 | "auto_awq", |
| 118 | "auto_gptq", |
| 119 | "inc", |
| 120 | "fp8", |
| 121 | "deepseek_v4_fp8", |
| 122 | "mxfp4", |
| 123 | "mxfp8", |
| 124 | "fp8_per_tensor", |
| 125 | "fp8_per_block", |
| 126 | "online", |
| 127 | "gpt_oss_mxfp4", |
| 128 | "modelopt", |
| 129 | "compressed-tensors", |
| 130 | ] |
| 131 | |
| 132 | @classmethod |
| 133 | def import_kernels(cls) -> None: |
| 134 | # Do not import vllm._C |
| 135 | with contextlib.suppress(ImportError): |
| 136 | import vllm._moe_C # noqa: F401 |
| 137 | |
| 138 | @classmethod |
| 139 | def check_runner_kv_caches_multi_layer(cls) -> None: |
| 140 | pass |
| 141 | |
| 142 | @classmethod |
| 143 | def get_attn_backend_cls( |
| 144 | cls, |
| 145 | selected_backend: "AttentionBackendEnum", |
| 146 | attn_selector_config: "AttentionSelectorConfig", |
| 147 | num_heads: int | None = None, |
| 148 | ) -> str: |
| 149 | # TurboQuant KV cache: route directly to TQ backend |
| 150 | kv_cache_dtype = attn_selector_config.kv_cache_dtype |
| 151 | if kv_cache_dtype is not None and kv_cache_dtype.startswith("turboquant_"): |
| 152 | logger.info_once("Using TurboQuant attention backend.") |
| 153 | return AttentionBackendEnum.TURBOQUANT.get_path() |
| 154 | |
| 155 | dtype = attn_selector_config.dtype |
| 156 | if attn_selector_config.use_sparse: |
| 157 | logger.info_once("Using XPU MLA Sparse backend.") |
| 158 | return AttentionBackendEnum.XPU_MLA_SPARSE.get_path() |
| 159 | if attn_selector_config.use_mla: |
| 160 | logger.info_once("Using Triton MLA backend on V1 engine.") |
| 161 | return AttentionBackendEnum.TRITON_MLA.get_path() |
| 162 | if selected_backend == AttentionBackendEnum.TRITON_ATTN: |
| 163 | logger.info_once("Using Triton backend.") |
| 164 | return AttentionBackendEnum.TRITON_ATTN.get_path() |
| 165 | elif attn_selector_config.use_batch_invariant: |
| 166 | # Flash Attention on XPU has not been validated for batch |
| 167 | # invariance. Honor an explicit Flash Attention request; |
| 168 | # otherwise fall back to Triton Attention, which implements |
| 169 | # batch-invariant kernels. |
| 170 | if selected_backend == AttentionBackendEnum.FLASH_ATTN: |
| 171 | logger.warning_once( |
| 172 | "Using Flash Attention on XPU with batch invariance " |
| 173 | "enabled because it was explicitly requested. This " |
| 174 | "backend has not been validated for batch invariance " |
| 175 | "on XPU and may produce non-deterministic results " |
| 176 | "across batch sizes." |
| 177 | ) |
| 178 | return AttentionBackendEnum.FLASH_ATTN.get_path() |
| 179 | logger.info_once( |
| 180 | "VLLM_BATCH_INVARIANT is enabled. Using Triton Attention " |
| 181 | "backend on XPU, which implements batch-invariant kernels." |
| 182 | ) |
| 183 | return AttentionBackendEnum.TRITON_ATTN.get_path() |
| 184 | elif attn_selector_config.use_mm_prefix: |
| 185 | # Flash Attention on XPU has no FA4 kernel, so it cannot apply the |
| 186 | # multimodal prefix-LM bidirectional mask. Honor an explicit Flash |
| 187 | # Attention request (for text-only workloads); otherwise fall back |
| 188 | # to Triton Attention, which supports mm_prefix. |
| 189 | if selected_backend == AttentionBackendEnum.FLASH_ATTN: |
| 190 | logger.warning_once( |
| 191 | "Using Flash Attention on XPU for a multimodal prefix-LM " |
| 192 | "model because it was explicitly requested. The prefix-LM " |
| 193 | "bidirectional mask cannot be applied, so image/video " |
| 194 | "inputs will produce incorrect results; only use this for " |
| 195 | "text-only workloads." |
| 196 | ) |
| 197 | return AttentionBackendEnum.FLASH_ATTN.get_path() |
| 198 | logger.warning_once( |
| 199 | "Flash Attention on XPU does not support multimodal prefix-LM " |
| 200 | "attention. Falling back to Triton Attention backend." |
| 201 | ) |
| 202 | return AttentionBackendEnum.TRITON_ATTN.get_path() |
| 203 | elif dtype == torch.float32: |
| 204 | logger.warning_once( |
| 205 | "Flash Attention on XPU does not support float32 dtype. " |
| 206 | "Falling back to Triton Attention backend." |
| 207 | ) |
| 208 | return AttentionBackendEnum.TRITON_ATTN.get_path() |
| 209 | elif selected_backend == AttentionBackendEnum.FLASH_ATTN: |
| 210 | logger.info_once("Using Flash Attention backend.") |
| 211 | return AttentionBackendEnum.FLASH_ATTN.get_path() |
| 212 | elif selected_backend: |
| 213 | raise ValueError( |
| 214 | f"Invalid attention backend for {cls.device_name}, " |
| 215 | f"with use_mla: {attn_selector_config.use_mla}" |
| 216 | ) |
| 217 | |
| 218 | logger.info_once("Using Flash Attention backend.") |
| 219 | return AttentionBackendEnum.FLASH_ATTN.get_path() |
| 220 | |
| 221 | @classmethod |
| 222 | def get_supported_vit_attn_backends(cls) -> list["AttentionBackendEnum"]: |
| 223 | return [ |
| 224 | AttentionBackendEnum.FLASH_ATTN, |
| 225 | AttentionBackendEnum.TRITON_ATTN, |
| 226 | AttentionBackendEnum.TORCH_SDPA, |
| 227 | ] |
| 228 | |
| 229 | @classmethod |
| 230 | def get_vit_attn_backend( |
| 231 | cls, |
| 232 | head_size: int, |
| 233 | dtype: torch.dtype, |
| 234 | backend: "AttentionBackendEnum | None" = None, |
| 235 | ) -> "AttentionBackendEnum": |
| 236 | if dtype == torch.float32: |
| 237 | logger.warning_once( |
| 238 | "Flash Attention on XPU does not support float32 dtype. " |
| 239 | "Falling back to Triton Attention backend for vit attention." |
| 240 | ) |
| 241 | return AttentionBackendEnum.TRITON_ATTN |
| 242 | |
| 243 | if backend is not None: |
| 244 | assert backend in cls.get_supported_vit_attn_backends(), ( |
| 245 | f"Backend {backend} is not supported for vit attention. " |
| 246 | f"Supported backends are: " |
| 247 | f"{cls.get_supported_vit_attn_backends()}." |
| 248 | ) |
| 249 | logger.info_once(f"Using backend {backend} for vit attention") |
| 250 | return backend |
| 251 | |
| 252 | logger.info_once( |
| 253 | f"Using backend {AttentionBackendEnum.FLASH_ATTN} for vit attention" |
| 254 | ) |
| 255 | return AttentionBackendEnum.FLASH_ATTN |
| 256 | |
| 257 | @classmethod |
| 258 | def set_device(cls, device: torch.device) -> None: |
| 259 | """Set the device for the current platform.""" |
| 260 | torch.xpu.set_device(device) |
| 261 | |
| 262 | @classmethod |
| 263 | def manual_seed_all(cls, seed: int) -> None: |
| 264 | torch.xpu.manual_seed_all(seed) |
| 265 | |
| 266 | @classmethod |
| 267 | def get_device_capability( |
| 268 | cls, |
| 269 | device_id: int = 0, |
| 270 | ) -> DeviceCapability | None: |
| 271 | # capacity format differs from cuda's and will cause unexpected |
| 272 | # failure, so use None directly |
| 273 | return None |
| 274 | |
| 275 | @classmethod |
| 276 | def get_device_name(cls, device_id: int = 0) -> str: |
| 277 | return torch.xpu.get_device_name(device_id) |
| 278 | |
| 279 | @classmethod |
| 280 | def get_punica_wrapper(cls) -> str: |
| 281 | xpu_use_triton_kernel = os.getenv("XPU_USE_TRITON_KERNEL", "0") == "1" |
| 282 | if not xpu_use_triton_kernel: |
| 283 | return "vllm.lora.punica_wrapper.punica_xpu.PunicaWrapperXPU" |
| 284 | else: |
| 285 | return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU" |
| 286 | |
| 287 | @classmethod |
| 288 | def get_device_total_memory(cls, device_id: int = 0) -> int: |
| 289 | device_props = torch.xpu.get_device_properties(device_id) |
| 290 | return device_props.total_memory |
| 291 | |
| 292 | @classmethod |
| 293 | def inference_mode(cls): |
| 294 | return torch.no_grad() |
| 295 | |
| 296 | @classmethod |
| 297 | def get_static_graph_wrapper_cls(cls) -> str: |
| 298 | return "vllm.compilation.cuda_graph.CUDAGraphWrapper" |
| 299 | |
| 300 | @classmethod |
| 301 | def check_and_update_config(cls, vllm_config: VllmConfig) -> None: |
| 302 | # lazy import to avoid circular import |
| 303 | from vllm.config import CUDAGraphMode |
| 304 | |
| 305 | if envs.VLLM_BATCH_INVARIANT: |
| 306 | model_config = vllm_config.model_config |
| 307 | if model_config is not None and ( |
| 308 | model_config.quantization is not None |
| 309 | or vllm_config.quant_config is not None |
| 310 | ): |
| 311 | raise ValueError( |
| 312 | "XPU batch invariance currently supports only unquantized " |
| 313 | f"models; got quantization={model_config.quantization!r}. " |
| 314 | "Use an unquantized model or disable VLLM_BATCH_INVARIANT." |
| 315 | ) |
| 316 | |
| 317 | cache_dtype = vllm_config.cache_config.cache_dtype |
| 318 | if cache_dtype not in ("auto", "float16", "bfloat16"): |
| 319 | raise ValueError( |
| 320 | "XPU batch invariance currently does not support quantized " |
| 321 | f"KV caches; got kv_cache_dtype={cache_dtype!r}. " |
| 322 | "Use an unquantized KV cache dtype or disable " |
| 323 | "VLLM_BATCH_INVARIANT." |
| 324 | ) |
| 325 | |
| 326 | compilation_config = vllm_config.compilation_config |
| 327 | if compilation_config.compile_sizes is None: |
| 328 | compilation_config.compile_sizes = [] |
| 329 | |
| 330 | # lazy import to avoid circular import |
| 331 | from vllm.utils.torch_utils import supports_xpu_graph |
| 332 | |
| 333 | if not supports_xpu_graph(): |
| 334 | compilation_config.cudagraph_mode = CUDAGraphMode.NONE |
| 335 | logger.warning_once( |
| 336 | "XPU Graph is not supported in the current PyTorch version, " |
| 337 | "disabling cudagraph_mode." |
| 338 | ) |
| 339 | |
| 340 | if ( |
| 341 | vllm_config.model_config is not None |
| 342 | and vllm_config.model_config.enable_sleep_mode |
| 343 | and compilation_config.cudagraph_mode != CUDAGraphMode.NONE |
| 344 | ): |
| 345 | compilation_config.cudagraph_mode = CUDAGraphMode.NONE |
| 346 | logger.warning_once( |
| 347 | "XPU Graph is not compatible with sleep mode, disabling cudagraph_mode." |
| 348 | ) |
| 349 | |
| 350 | # Disable fusion passes not yet supported on XPU. |
| 351 | from vllm.config.compilation import CompilationMode |
| 352 | |
| 353 | pass_config = compilation_config.pass_config |
| 354 | fusion_passes_to_disable = { |
| 355 | "fuse_gemm_comms": "Async TP", |
| 356 | "fuse_allreduce_rms": "AllReduce + RMSNorm fusion", |
| 357 | "fuse_attn_quant": "Attention + quant fusion", |
| 358 | "fuse_act_padding": "Activation + padding fusion", |
| 359 | "fuse_rope_kvcache": "RoPE + KV cache fusion", |
| 360 | "fuse_rope_kvcache_cat_mla": "RoPE + KV cache + MLA fusion", |
| 361 | } |
| 362 | if compilation_config.mode != CompilationMode.NONE: |
| 363 | for flag, feature_name in fusion_passes_to_disable.items(): |
| 364 | if getattr(pass_config, flag): |
| 365 | logger.warning_once( |
| 366 | "Feature %r is not yet supported on XPU and will be disabled.", |
| 367 | feature_name, |
| 368 | ) |
| 369 | setattr(pass_config, flag, False) |
| 370 | |
| 371 | # UVA-offloaded weights are host USM allocations, which Inductor's |
| 372 | # static Triton launcher rejects ("Pointer argument doesn't reference |
| 373 | # XPU device memory"). Fall back to Triton's own launcher. Remove once |
| 374 | # the released torch contains pytorch/pytorch#188240, which relaxes |
| 375 | # that check to any memory type known by the driver. |
| 376 | offload_config = vllm_config.offload_config |
| 377 | uva_offloading = offload_config.offload_backend == "uva" or ( |
| 378 | offload_config.offload_backend == "auto" |
| 379 | and offload_config.prefetch.offload_group_size == 0 |
| 380 | and offload_config.uva.cpu_offload_gb > 0 |
| 381 | ) |
| 382 | if ( |
| 383 | uva_offloading |
| 384 | and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_UVA |
| 385 | and compilation_config.mode != CompilationMode.NONE |
| 386 | ): |
| 387 | compilation_config.inductor_compile_config.setdefault( |
| 388 | "use_static_cuda_launcher", False |
| 389 | ) |
| 390 | logger.info_once( |
| 391 | "Disabling Inductor's static Triton launcher because UVA " |
| 392 | "weight offloading is enabled." |
| 393 | ) |
| 394 | |
| 395 | # check and update parallel config |
| 396 | parallel_config = vllm_config.parallel_config |
| 397 | # Only override worker_cls if it's still the default "auto" |
| 398 | # This allows custom workers (like vllm-omni workers) to be used on XPU |
| 399 | if parallel_config.worker_cls == "auto": |
| 400 | parallel_config.worker_cls = "vllm.v1.worker.xpu_worker.XPUWorker" |
| 401 | if vllm_config.kv_transfer_config is not None: |
| 402 | vllm_config.kv_transfer_config.enable_permute_local_kv = True |
| 403 | |
| 404 | # In some cases, the internal memory type cache can misdetect GPU |
| 405 | # memory as host memory, also leading to invalid memory access. |
| 406 | # This cache can be disabled by setting UCX_MEMTYPE_CACHE=n. |
| 407 | # ref. https://openucx.readthedocs.io/en/master/faq.html |
| 408 | os.environ["UCX_MEMTYPE_CACHE"] = "n" |
| 409 | |
| 410 | # spawn is the only supported multiprocessing method on XPU |
| 411 | if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ: |
| 412 | os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" |
| 413 | |
| 414 | # XPU requires graceful shutdown to allow oneCCL/Level Zero resources |
| 415 | # to be properly released. Without this, subsequent server startups on |
| 416 | # the same devices may hang during CCL initialization. |
| 417 | if vllm_config.shutdown_timeout == 0: |
| 418 | vllm_config.shutdown_timeout = 5 |
| 419 | logger.info( |
| 420 | "XPU platform: set server shutdown_timeout=%d.", |
| 421 | vllm_config.shutdown_timeout, |
| 422 | ) |
| 423 | |
| 424 | @classmethod |
| 425 | def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: |
| 426 | super().update_block_size_for_backend(vllm_config) |
| 427 | from vllm.config.vllm import get_layers_from_vllm_config |
| 428 | from vllm.model_executor.layers.attention_layer_base import ( |
| 429 | AttentionLayerBase, |
| 430 | ) |
| 431 | from vllm.utils.math_utils import cdiv |
| 432 | |
| 433 | cache_config = vllm_config.cache_config |
| 434 | # special fix for GDN since kernel only supports block size dividable by 64 |
| 435 | attn_layers = get_layers_from_vllm_config( |
| 436 | vllm_config, |
| 437 | AttentionLayerBase, # type: ignore[type-abstract] |
| 438 | ) |
| 439 | |
| 440 | kernel_block_size = None |
| 441 | for layer in attn_layers.values(): |
| 442 | b = layer.get_attn_backend() |
| 443 | if b.get_name() == "GDN_ATTN": |
| 444 | kernel_block_size = 64 |
| 445 | break |
| 446 | |
| 447 | if kernel_block_size is None: |
| 448 | return |
| 449 | new_block_size = ( |
| 450 | cdiv(cache_config.block_size, kernel_block_size) * kernel_block_size |
| 451 | ) |
| 452 | if new_block_size == cache_config.block_size: |
| 453 | return |
| 454 | |
| 455 | if cache_config.mamba_cache_mode == "align": |
| 456 | cache_config.mamba_block_size = new_block_size |
| 457 | original_mamba_page_size_padded = cache_config.mamba_page_size_padded |
| 458 | if cache_config.mamba_page_size_padded is not None: |
| 459 | attn_page_size_1_token = ( |
| 460 | cache_config.mamba_page_size_padded // cache_config.block_size |
| 461 | ) |
| 462 | cache_config.mamba_page_size_padded = ( |
| 463 | new_block_size * attn_page_size_1_token |
| 464 | ) |
| 465 | cache_config.block_size = new_block_size |
| 466 | logger.info( |
| 467 | "[XPU]Setting attention block size to %d tokens to ensure multiple of %d.", |
| 468 | new_block_size, |
| 469 | kernel_block_size, |
| 470 | ) |
| 471 | if original_mamba_page_size_padded is not None: |
| 472 | logger.info( |
| 473 | "[XPU]Scaled mamba_page_size_padded from %d to %d bytes accordingly.", |
| 474 | original_mamba_page_size_padded, |
| 475 | cache_config.mamba_page_size_padded, |
| 476 | ) |
| 477 | |
| 478 | @classmethod |
| 479 | def support_hybrid_kv_cache(cls) -> bool: |
| 480 | return True |
| 481 | |
| 482 | @classmethod |
| 483 | def support_static_graph_mode(cls) -> bool: |
| 484 | return True |
| 485 | |
| 486 | @classmethod |
| 487 | def is_pin_memory_available(cls): |
| 488 | return True |
| 489 | |
| 490 | @classmethod |
| 491 | def get_current_memory_usage( |
| 492 | cls, device: torch.types.Device | None = None |
| 493 | ) -> float: |
| 494 | torch.xpu.empty_cache() |
| 495 | torch.xpu.reset_peak_memory_stats(device) |
| 496 | return torch.xpu.max_memory_allocated(device) |
| 497 | |
| 498 | @classmethod |
| 499 | def fp8_dtype(cls) -> torch.dtype: |
| 500 | return torch.float8_e4m3fn |
| 501 | |
| 502 | @classmethod |
| 503 | def is_data_center_gpu(cls) -> bool: |
| 504 | device_name = cls.get_device_name().lower() |
| 505 | return device_name.count("data center gpu") > 0 |
| 506 | |
| 507 | @classmethod |
| 508 | def get_device_communicator_cls(cls) -> str: |
| 509 | if not torch.distributed.is_xccl_available(): |
| 510 | # Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform |
| 511 | logger.warning( |
| 512 | "xccl is not enabled in this torch build, communication" |
| 513 | " is not available." |
| 514 | ) |
| 515 | return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator" # noqa |
| 516 | |
| 517 | @classmethod |
| 518 | def supports_fp8(cls) -> bool: |
| 519 | return True |
| 520 | |
| 521 | @classmethod |
| 522 | def get_default_ir_op_priority( |
| 523 | cls, vllm_config: "VllmConfig" |
| 524 | ) -> "IrOpPriorityConfig": |
| 525 | from vllm.config.compilation import CompilationMode |
| 526 | from vllm.config.kernel import IrOpPriorityConfig |
| 527 | |
| 528 | # Native used by default when compiling, |
| 529 | # use fused kernels where available when no codegen |
| 530 | cc = vllm_config.compilation_config |
| 531 | using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE |
| 532 | default = ["native"] if using_inductor else ["vllm_c", "native"] |
| 533 | |
| 534 | return IrOpPriorityConfig.with_default(default, gelu_and_mul_sparse=["native"]) |
| 535 | |
| 536 | @classmethod |
| 537 | def device_count(cls) -> int: |
| 538 | return torch.xpu.device_count() |
| 539 | |
| 540 | @classmethod |
| 541 | def check_if_supports_dtype(cls, dtype: torch.dtype): |
| 542 | if dtype == torch.bfloat16: # noqa: SIM102 |
| 543 | device_name = cls.get_device_name().lower() |
| 544 | # client gpu a770 |
| 545 | if device_name.count("a770") > 0: |
| 546 | raise ValueError( |
| 547 | "Intel Arc A770 have bfloat16 accuracy known issue. " |
| 548 | "You can use float16 instead by explicitly setting the " |
| 549 | "`dtype` flag in CLI, for example: --dtype=half." |
| 550 | ) |
| 551 | |
| 552 | @classmethod |
| 553 | def opaque_attention_op(cls) -> bool: |
| 554 | return True |
| 555 | |
| 556 | @classmethod |
| 557 | def insert_blocks_to_device( |
| 558 | cls, |
| 559 | src_cache: torch.Tensor, |
| 560 | dst_cache: torch.Tensor, |
| 561 | src_block_indices: torch.Tensor, |
| 562 | dst_block_indices: torch.Tensor, |
| 563 | ) -> None: |
| 564 | """Copy blocks from src_cache to dst_cache on XPU.""" |
| 565 | _src_cache = src_cache[src_block_indices] |
| 566 | dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device) |
| 567 | |
| 568 | @classmethod |
| 569 | def swap_out_blocks_to_host( |
| 570 | cls, |
| 571 | src_cache: torch.Tensor, |
| 572 | dst_cache: torch.Tensor, |
| 573 | src_block_indices: torch.Tensor, |
| 574 | dst_block_indices: torch.Tensor, |
| 575 | ) -> None: |
| 576 | """Copy blocks from XPU to host (CPU).""" |
| 577 | _src_cache = src_cache[src_block_indices] |
| 578 | dst_cache[dst_block_indices] = _src_cache.cpu() |
| 579 | |
| 580 | @classmethod |
| 581 | def num_compute_units(cls, device_id: int = 0) -> int: |
| 582 | return torch.xpu.get_device_properties(device_id).max_compute_units |
| 583 | |
| 584 | @classmethod |
| 585 | def use_custom_op_collectives(cls) -> bool: |
| 586 | return True |