@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04a3ae0aca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix multi nodes TP+PP for XPU (#8884)

Signed-off-by: YiSheng5 <syhm@mail.ustc.edu.cn> Signed-off-by: yan ma <yan.ma@intel.com> Co-authored-by: YiSheng5 <syhm@mail.ustc.edu.cn>

Yan Ma committed Oct 30, 2024 at 12:34 UTC 04a3ae0acae3d522299ec90b5730f876daa845e6
7 files changed +63 -11
docs/source/getting_started/xpu-installation.rst
+18
@@ -60,3 +60,21 @@ Build from source
60 - FP16 is the default data type in the current XPU backend. The BF16 data
61 type will be supported in the future.
62
63 +
64 +Distributed inference and serving
65 +---------------------------------
66 +
67 +XPU platform supports tensor-parallel inference/serving and also supports pipeline parallel as a beta feature for online serving. We requires Ray as the distributed runtime backend. For example, a reference execution likes following:
68 +
69 +.. code-block:: console
70 +
71 + $ python -m vllm.entrypoints.openai.api_server \
72 + $ --model=facebook/opt-13b \
73 + $ --dtype=bfloat16 \
74 + $ --device=xpu \
75 + $ --max_model_len=1024 \
76 + $ --distributed-executor-backend=ray \
77 + $ --pipeline-parallel-size=2 \
78 + $ -tp=8
79 +
80 +By default, a ray instance will be launched automatically if no existing one is detected in system, with ``num-gpus`` equals to ``parallel_config.world_size``. We recommend properly starting a ray cluster before execution, referring helper `script <https://github.com/vllm-project/vllm/tree/main/examples/run_cluster.sh>`_.
requirements-xpu.txt
+1 -1
@@ -13,4 +13,4 @@ torch == 2.3.1+cxx11.abi
13 intel-extension-for-pytorch == 2.3.110+xpu
14 oneccl_bind_pt == 2.3.100+xpu
15
16 -triton-xpu == 3.0.0b2
16 +triton-xpu == 3.0.0b1
vllm/distributed/parallel_state.py
+22
@@ -431,6 +431,28 @@ class GroupCoordinator:
431 if dim < 0:
432 # Convert negative dim to positive.
433 dim += input_.dim()
434 + # For xpu path, gather doesn't work properly together with ray
435 + # cluster so we use all_gather instead for now.
436 + if current_platform.is_xpu():
437 + input_size = input_.size()
438 + # Allocate output tensor.
439 + output_tensor = torch.empty((world_size, ) + input_size,
440 + dtype=input_.dtype,
441 + device=input_.device)
442 + # All-gather.
443 + torch.distributed.all_gather_into_tensor(output_tensor,
444 + input_,
445 + group=self.device_group)
446 + if self.rank_in_group == dst:
447 + # Reshape
448 + output_tensor = output_tensor.movedim(0, dim)
449 + output_tensor = output_tensor.reshape(input_size[:dim] +
450 + (world_size *
451 + input_size[dim], ) +
452 + input_size[dim + 1:])
453 + else:
454 + output_tensor = None
455 + return output_tensor
456 # Allocate output tensor.
457 if self.rank_in_group == dst:
458 gather_list = [torch.empty_like(input_) for _ in range(world_size)]
vllm/executor/xpu_executor.py
+11 -1
@@ -44,7 +44,7 @@ class XPUExecutor(GPUExecutor):
44 self.cache_config = cache_config
45 self.load_config = load_config
46 self.lora_config = lora_config
47 - self.parallel_config = parallel_config
47 + self.parallel_config = _verify_and_get_parallel_config(parallel_config)
48 self.scheduler_config = scheduler_config
49 self.device_config = device_config
50 self.prompt_adapter_config = prompt_adapter_config
@@ -94,3 +94,13 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
94 "mode.")
95 config.enforce_eager = True
96 return config
97 +
98 +
99 +def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
100 + if (config.distributed_executor_backend is not None
101 + and config.distributed_executor_backend != "ray"):
102 + logger.warning(
103 + "%s is not supported on XPU, fallback to ray distributed executor "
104 + "backend.", config.distributed_executor_backend)
105 + config.distributed_executor_backend = "ray"
106 + return config
vllm/platforms/__init__.py
+3
@@ -45,6 +45,9 @@ except Exception:
45 is_xpu = False
46
47 try:
48 + # installed IPEX if the machine has XPUs.
49 + import intel_extension_for_pytorch # noqa: F401
50 + import oneccl_bindings_for_pytorch # noqa: F401
51 import torch
52 if hasattr(torch, 'xpu') and torch.xpu.is_available():
53 is_xpu = True
vllm/platforms/xpu.py
+4
@@ -20,3 +20,7 @@ class XPUPlatform(Platform):
20 def get_device_total_memory(cls, device_id: int = 0) -> int:
21 device_props = torch.xpu.get_device_properties(device_id)
22 return device_props.total_memory
23 +
24 + @staticmethod
25 + def inference_mode():
26 + return torch.no_grad()
vllm/worker/xpu_worker.py
+4 -9
@@ -14,7 +14,6 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14 SpeculativeConfig)
15 from vllm.distributed import (ensure_model_parallel_initialized,
16 init_distributed_environment)
17 -from vllm.distributed.parallel_state import get_pp_group
17 from vllm.logger import init_logger
18 from vllm.model_executor import set_random_seed
19 from vllm.platforms import current_platform
@@ -183,11 +182,10 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
182 # use sockets as default Level zero IPC exchange backend. By
183 # default oneccl will use `drmfd` as mechanism which need extra
184 # dependency (libdrm and drm headers) on your system.
186 - ENV_CCL_ZE_IPC_EXCHANGE = os.getenv("CCL_ZE_IPC_EXCHANGE",
187 - "sockets")
185 + ENV_CCL_ATL_TRANSPORT = os.getenv("CCL_ATL_TRANSPORT", "ofi")
186 ENV_LOCAL_WORLD_SIZE = os.getenv("LOCAL_WORLD_SIZE",
187 str(parallel_config.world_size))
190 - os.environ['CCL_ZE_IPC_EXCHANGE'] = ENV_CCL_ZE_IPC_EXCHANGE
188 + os.environ["CCL_ATL_TRANSPORT"] = ENV_CCL_ATL_TRANSPORT
189 os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE
190 os.environ["LOCAL_RANK"] = str(self.local_rank)
191 init_distributed_environment(
@@ -200,8 +198,5 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
198 ensure_model_parallel_initialized(
199 parallel_config.tensor_parallel_size,
200 parallel_config.pipeline_parallel_size)
203 -
204 - if parallel_config.pipeline_parallel_size > 1:
205 - # torch-ccl xpu need a collective API warm up
206 - # before calling send/recv API
207 - get_pp_group().all_reduce(torch.zeros(1).xpu())
201 + # global all_reduce needed for overall oneccl warm up
202 + torch.distributed.all_reduce(torch.zeros(1).xpu())