A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix multi nodes TP+PP for XPU (#8884)
Signed-off-by: YiSheng5 <syhm@mail.ustc.edu.cn> Signed-off-by: yan ma <yan.ma@intel.com> Co-authored-by: YiSheng5 <syhm@mail.ustc.edu.cn>
Yan Ma committed
Oct 30, 2024 at 12:34 UTC
04a3ae0acae3d522299ec90b5730f876daa845e6
7 files changed
+63
-11
docs/source/getting_started/xpu-installation.rst
+18
@@ -60,3 +60,21 @@ Build from source
60
- FP16 is the default data type in the current XPU backend. The BF16 data
61
type will be supported in the future.
62
63
+
64
+Distributed inference and serving
65
+---------------------------------
66
+
67
+XPU platform supports tensor-parallel inference/serving and also supports pipeline parallel as a beta feature for online serving. We requires Ray as the distributed runtime backend. For example, a reference execution likes following:
68
+
69
+.. code-block:: console
70
+
71
+ $ python -m vllm.entrypoints.openai.api_server \
72
+ $ --model=facebook/opt-13b \
73
+ $ --dtype=bfloat16 \
74
+ $ --device=xpu \
75
+ $ --max_model_len=1024 \
76
+ $ --distributed-executor-backend=ray \
77
+ $ --pipeline-parallel-size=2 \
78
+ $ -tp=8
79
+
80
+By default, a ray instance will be launched automatically if no existing one is detected in system, with ``num-gpus`` equals to ``parallel_config.world_size``. We recommend properly starting a ray cluster before execution, referring helper `script <https://github.com/vllm-project/vllm/tree/main/examples/run_cluster.sh>`_.
requirements-xpu.txt
+1
-1
@@ -13,4 +13,4 @@ torch == 2.3.1+cxx11.abi
13
intel-extension-for-pytorch == 2.3.110+xpu
14
oneccl_bind_pt == 2.3.100+xpu
15
16
-triton-xpu == 3.0.0b2
16
+triton-xpu == 3.0.0b1
vllm/distributed/parallel_state.py
+22
@@ -431,6 +431,28 @@ class GroupCoordinator:
431
if dim < 0:
432
# Convert negative dim to positive.
433
dim += input_.dim()
434
+ # For xpu path, gather doesn't work properly together with ray
435
+ # cluster so we use all_gather instead for now.
436
+ if current_platform.is_xpu():
437
+ input_size = input_.size()
438
+ # Allocate output tensor.
439
+ output_tensor = torch.empty((world_size, ) + input_size,
440
+ dtype=input_.dtype,
441
+ device=input_.device)
442
+ # All-gather.
443
+ torch.distributed.all_gather_into_tensor(output_tensor,
444
+ input_,
445
+ group=self.device_group)
446
+ if self.rank_in_group == dst:
447
+ # Reshape
448
+ output_tensor = output_tensor.movedim(0, dim)
449
+ output_tensor = output_tensor.reshape(input_size[:dim] +
450
+ (world_size *
451
+ input_size[dim], ) +
452
+ input_size[dim + 1:])
453
+ else:
454
+ output_tensor = None
455
+ return output_tensor
456
# Allocate output tensor.
457
if self.rank_in_group == dst:
458
gather_list = [torch.empty_like(input_) for _ in range(world_size)]
vllm/executor/xpu_executor.py
+11
-1
@@ -44,7 +44,7 @@ class XPUExecutor(GPUExecutor):
44
self.cache_config = cache_config
45
self.load_config = load_config
46
self.lora_config = lora_config
47
- self.parallel_config = parallel_config
47
+ self.parallel_config = _verify_and_get_parallel_config(parallel_config)
48
self.scheduler_config = scheduler_config
49
self.device_config = device_config
50
self.prompt_adapter_config = prompt_adapter_config
@@ -94,3 +94,13 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
94
"mode.")
95
config.enforce_eager = True
96
return config
97
+
98
+
99
+def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
100
+ if (config.distributed_executor_backend is not None
101
+ and config.distributed_executor_backend != "ray"):
102
+ logger.warning(
103
+ "%s is not supported on XPU, fallback to ray distributed executor "
104
+ "backend.", config.distributed_executor_backend)
105
+ config.distributed_executor_backend = "ray"
106
+ return config
vllm/platforms/__init__.py
+3
@@ -45,6 +45,9 @@ except Exception:
45
is_xpu = False
46
47
try:
48
+ # installed IPEX if the machine has XPUs.
49
+ import intel_extension_for_pytorch # noqa: F401
50
+ import oneccl_bindings_for_pytorch # noqa: F401
51
import torch
52
if hasattr(torch, 'xpu') and torch.xpu.is_available():
53
is_xpu = True
vllm/platforms/xpu.py
+4
@@ -20,3 +20,7 @@ class XPUPlatform(Platform):
20
def get_device_total_memory(cls, device_id: int = 0) -> int:
21
device_props = torch.xpu.get_device_properties(device_id)
22
return device_props.total_memory
23
+
24
+ @staticmethod
25
+ def inference_mode():
26
+ return torch.no_grad()
vllm/worker/xpu_worker.py
+4
-9
@@ -14,7 +14,6 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14
SpeculativeConfig)
15
from vllm.distributed import (ensure_model_parallel_initialized,
16
init_distributed_environment)
17
-from vllm.distributed.parallel_state import get_pp_group
17
from vllm.logger import init_logger
18
from vllm.model_executor import set_random_seed
19
from vllm.platforms import current_platform
@@ -183,11 +182,10 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
182
# use sockets as default Level zero IPC exchange backend. By
183
# default oneccl will use `drmfd` as mechanism which need extra
184
# dependency (libdrm and drm headers) on your system.
186
- ENV_CCL_ZE_IPC_EXCHANGE = os.getenv("CCL_ZE_IPC_EXCHANGE",
187
- "sockets")
185
+ ENV_CCL_ATL_TRANSPORT = os.getenv("CCL_ATL_TRANSPORT", "ofi")
186
ENV_LOCAL_WORLD_SIZE = os.getenv("LOCAL_WORLD_SIZE",
187
str(parallel_config.world_size))
190
- os.environ['CCL_ZE_IPC_EXCHANGE'] = ENV_CCL_ZE_IPC_EXCHANGE
188
+ os.environ["CCL_ATL_TRANSPORT"] = ENV_CCL_ATL_TRANSPORT
189
os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE
190
os.environ["LOCAL_RANK"] = str(self.local_rank)
191
init_distributed_environment(
@@ -200,8 +198,5 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
198
ensure_model_parallel_initialized(
199
parallel_config.tensor_parallel_size,
200
parallel_config.pipeline_parallel_size)
203
-
204
- if parallel_config.pipeline_parallel_size > 1:
205
- # torch-ccl xpu need a collective API warm up
206
- # before calling send/recv API
207
- get_pp_group().all_reduce(torch.zeros(1).xpu())
201
+ # global all_reduce needed for overall oneccl warm up
202
+ torch.distributed.all_reduce(torch.zeros(1).xpu())