@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e2c19ce22

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Refactor]Abstract Platform Interface for Distributed Backend and Add xccl Support for Intel XPU (#19410)

Signed-off-by: dbyoung18 <yang5.yang@intel.com> Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>

Yang Yang committed Jul 7, 2025 at 12:32 UTC 6e2c19ce227ecf285ed24a138b91570b3a2d57a6

17 files changed +44 -8

docs/getting_started/installation/gpu/xpu.inc.md

@@ -81,4 +81,9 @@ python -m vllm.entrypoints.openai.api_server \

81 By default, a ray instance will be launched automatically if no existing one is detected in the system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring to the <gh-file:examples/online_serving/run_cluster.sh> helper script.

83 # --8<-- [end:supported-features]

84 +# --8<-- [start:distributed-backend]

85 +

86 +XPU platform uses **torch-ccl** for torch<2.8 and **xccl** for torch>=2.8 as distributed backend, since torch 2.8 supports **xccl** as built-in backend for XPU.

87 +

88 +# --8<-- [end:distributed-backend]

89 # --8<-- [end:extra-information]

vllm/platforms/__init__.py

+11 -2

@@ -7,7 +7,7 @@ from itertools import chain

7 from typing import TYPE_CHECKING, Optional

9 from vllm.plugins import load_plugins_by_group

10 -from vllm.utils import resolve_obj_by_qualname

10 +from vllm.utils import resolve_obj_by_qualname, supports_xccl

12 from .interface import _Backend # noqa: F401

13 from .interface import CpuArchEnum, Platform, PlatformEnum

@@ -139,10 +139,19 @@ def xpu_platform_plugin() -> Optional[str]:

139 try:

140 # installed IPEX if the machine has XPUs.

141 import intel_extension_for_pytorch # noqa: F401

142 - import oneccl_bindings_for_pytorch # noqa: F401

142 import torch

143 + if supports_xccl():

144 + dist_backend = "xccl"

145 + else:

146 + dist_backend = "ccl"

147 + import oneccl_bindings_for_pytorch # noqa: F401

148 +

149 if hasattr(torch, 'xpu') and torch.xpu.is_available():

150 is_xpu = True

151 + from vllm.platforms.xpu import XPUPlatform

152 + XPUPlatform.dist_backend = dist_backend

153 + logger.debug("Confirmed %s backend is available.",

154 + XPUPlatform.dist_backend)

155 logger.debug("Confirmed XPU platform is available.")

156 except Exception as e:

157 logger.debug("XPU platform is not available because: %s", str(e))

vllm/platforms/cpu.py

@@ -37,6 +37,7 @@ class CpuPlatform(Platform):

37 device_name: str = "cpu"

38 device_type: str = "cpu"

39 dispatch_key: str = "CPU"

40 + dist_backend: str = "gloo"

42 @property

43 def supported_dtypes(self) -> list[torch.dtype]:

vllm/platforms/cuda.py

@@ -56,6 +56,7 @@ class CudaPlatformBase(Platform):

56 device_type: str = "cuda"

57 dispatch_key: str = "CUDA"

58 ray_device_key: str = "GPU"

59 + dist_backend: str = "nccl"

60 device_control_env_var: str = "CUDA_VISIBLE_DEVICES"

62 @property

vllm/platforms/hpu.py

@@ -26,6 +26,7 @@ class HpuPlatform(Platform):

26 device_type: str = "hpu"

27 dispatch_key: str = "HPU"

28 ray_device_key: str = "HPU"

29 + dist_backend: str = "hccl"

30 device_control_env_var: str = "HABANA_VISIBLE_MODULES"

32 @classmethod

vllm/platforms/interface.py

@@ -129,6 +129,9 @@ class Platform:

129 # compilation strategy.

130 simple_compile_backend: str = "inductor"

131

132 + # The backend used for distributed communication.

133 + dist_backend: str = ""

134 +

135 supported_quantization: list[str] = []

136

137 additional_env_vars: list[str] = []

vllm/platforms/neuron.py

@@ -30,6 +30,7 @@ class NeuronPlatform(Platform):

30 device_type: str = "neuron"

31 ray_device_key: str = "neuron_cores"

32 supported_quantization: list[str] = ["neuron_quant", "fbgemm_fp8"]

33 + dist_backend: str = "gloo"

34 device_control_env_var: str = "NEURON_RT_VISIBLE_CORES"

36 @classmethod

vllm/platforms/rocm.py

@@ -164,6 +164,7 @@ class RocmPlatform(Platform):

164 device_type: str = "cuda"

165 dispatch_key: str = "CUDA"

166 ray_device_key: str = "GPU"

167 + dist_backend: str = "nccl"

168 # rocm shares the same device control env var as CUDA

169 device_control_env_var: str = "CUDA_VISIBLE_DEVICES"

170

vllm/platforms/tpu.py

@@ -31,6 +31,7 @@ class TpuPlatform(Platform):

31 device_type: str = "tpu"

32 dispatch_key: str = "XLA"

33 ray_device_key: str = "TPU"

34 + dist_backend: str = "gloo"

35 device_control_env_var: str = "TPU_VISIBLE_CHIPS"

36 simple_compile_backend: str = "openxla"

vllm/platforms/xpu.py

@@ -29,6 +29,7 @@ class XPUPlatform(Platform):

29 # Intel XPU's device key is "GPU" for Ray.

30 # see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501

31 ray_device_key: str = "GPU"

32 + dist_backend: str = "ccl" # ccl | xccl

33 device_control_env_var: str = "ONEAPI_DEVICE_SELECTOR"

35 @classmethod

vllm/utils/__init__.py

@@ -1886,6 +1886,12 @@ def supports_dynamo() -> bool:

1886 return base_torch_version >= Version("2.4.0")

1887

1888

1889 +# Supports xccl with PyTorch versions >= 2.8.0 for XPU platform

1890 +def supports_xccl() -> bool:

1891 + return is_torch_equal_or_newer(

1892 + "2.8.0") and torch.distributed.is_xccl_available()

1893 +

1894 +

1895 # Some backends use pytorch version < 2.4.0 which doesn't

1896 # support `torch.library.custom_op`.

1897 def supports_custom_op() -> bool:

vllm/v1/worker/cpu_worker.py

+3 -1

@@ -11,6 +11,7 @@ from vllm.config import VllmConfig

11 from vllm.distributed.parallel_state import get_pp_group, get_tp_group

12 from vllm.logger import init_logger

13 from vllm.model_executor.utils import set_random_seed

14 +from vllm.platforms import current_platform

15 from vllm.sequence import IntermediateTensors

16 from vllm.v1.core.sched.output import SchedulerOutput

17 from vllm.v1.outputs import ModelRunnerOutput

@@ -58,7 +59,8 @@ class CPUWorker(Worker):

59 # Initialize the distributed environment.

60 init_worker_distributed_environment(self.vllm_config, self.rank,

61 self.distributed_init_method,

61 - self.local_rank, "gloo")

62 + self.local_rank,

63 + current_platform.dist_backend)

64 # Set random seed.

65 set_random_seed(self.model_config.seed)

vllm/v1/worker/gpu_worker.py

+2 -1

@@ -157,7 +157,8 @@ class Worker(WorkerBase):

157 # Initialize the distributed environment.

158 init_worker_distributed_environment(self.vllm_config, self.rank,

159 self.distributed_init_method,

160 - self.local_rank)

160 + self.local_rank,

161 + current_platform.dist_backend)

162 # Set random seed.

163 set_random_seed(self.model_config.seed)

164

vllm/v1/worker/tpu_worker.py

+2 -1

@@ -18,6 +18,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,

18 from vllm.logger import init_logger

19 from vllm.lora.request import LoRARequest

20 from vllm.model_executor import set_random_seed

21 +from vllm.platforms import current_platform

22 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, cdiv

23 from vllm.v1.attention.backends.pallas import TPU_HEAD_SIZE_ALIGNMENT

24 from vllm.v1.core.sched.output import SchedulerOutput

@@ -300,7 +301,7 @@ class TPUWorker:

301 rank=rank,

302 local_rank=local_rank,

303 distributed_init_method=distributed_init_method,

303 - backend="gloo",

304 + backend=current_platform.dist_backend,

305 )

306 ensure_model_parallel_initialized(

307 parallel_config.tensor_parallel_size,

vllm/worker/hpu_worker.py

+2 -1

@@ -23,6 +23,7 @@ from vllm.logger import init_logger

23 from vllm.lora.request import LoRARequest

24 from vllm.model_executor import set_random_seed

25 from vllm.model_executor.layers.sampler import SamplerOutput

26 +from vllm.platforms import current_platform

27 from vllm.prompt_adapter.request import PromptAdapterRequest

28 from vllm.sequence import ExecuteModelRequest

29 from vllm.utils import bind_kv_cache

@@ -413,7 +414,7 @@ def init_worker_distributed_environment(

414 rank,

415 distributed_init_method,

416 local_rank,

416 - backend='hccl')

417 + backend=current_platform.dist_backend)

418

419 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,

420 parallel_config.pipeline_parallel_size)

vllm/worker/neuron_worker.py

+1 -1

@@ -156,7 +156,7 @@ class NeuronWorker(LocalOrDistributedWorkerBase):

156 rank=self.rank,

157 local_rank=self.local_rank,

158 distributed_init_method=self.distributed_init_method,

159 - backend="gloo",

159 + backend=current_platform.dist_backend,

160 )

161

162 ensure_model_parallel_initialized(

vllm/worker/worker.py

+2 -1

@@ -530,7 +530,8 @@ def init_worker_distributed_environment(

530 set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)

531

532 init_distributed_environment(parallel_config.world_size, rank,

533 - distributed_init_method, local_rank)

533 + distributed_init_method, local_rank,

534 + current_platform.dist_backend)

535 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,

536 parallel_config.pipeline_parallel_size)

537