A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Distributed] use cpu/gloo to initialize pynccl (#4248)
youkaichao committed
Apr 23, 2024 at 18:32 UTC
91f50a6fe240b2c92a99e171bb11d083f82e4a84
5 files changed
+93
-71
tests/distributed/test_pynccl.py
+10
-5
@@ -5,6 +5,7 @@ import torch
5
6
from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
7
ncclGetUniqueId)
8
+from vllm.distributed.parallel_state import init_distributed_environment
9
from vllm.utils import update_environment_variables
10
11
@@ -26,19 +27,23 @@ def distributed_run(fn, world_size):
27
for p in processes:
28
p.join()
29
30
+ for p in processes:
31
+ assert p.exitcode == 0
32
+
33
30
-def update_env(fn):
34
+def worker_fn_wrapper(fn):
35
# `multiprocessing.Process` cannot accept environment variables directly
36
# so we need to pass the environment variables as arguments
37
# and update the environment variables in the function
34
- def wrapper(env):
38
+ def wrapped_fn(env):
39
update_environment_variables(env)
40
+ init_distributed_environment()
41
fn()
42
38
- return wrapper
43
+ return wrapped_fn
44
45
41
-@update_env
46
+@worker_fn_wrapper
47
def worker_fn():
48
comm = NCCLCommunicator()
49
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
@@ -53,7 +58,7 @@ def test_pynccl():
58
distributed_run(worker_fn, 2)
59
60
56
-@update_env
61
+@worker_fn_wrapper
62
def worker_fn_with_cudagraph():
63
with torch.no_grad():
64
graph = torch.cuda.CUDAGraph()
vllm/distributed/device_communicators/pynccl.py
+71
-51
@@ -20,14 +20,15 @@
20
# variable in the code.
21
22
import ctypes
23
-import datetime
23
import platform
24
+from typing import Optional, Union
25
26
# ===================== import region =====================
27
import torch
28
import torch.distributed as dist
29
-from torch.distributed import ReduceOp
29
+from torch.distributed import ProcessGroup, ReduceOp
30
31
+from vllm.distributed.parallel_state import get_cpu_world_group, get_local_rank
32
from vllm.logger import init_logger
33
from vllm.utils import find_nccl_library, nccl_integrity_check
34
@@ -59,6 +60,18 @@ except Exception as e:
60
61
ncclResult_t = ctypes.c_int
62
63
+_c_ncclGetErrorString = nccl.ncclGetErrorString
64
+_c_ncclGetErrorString.restype = ctypes.c_char_p
65
+_c_ncclGetErrorString.argtypes = [ncclResult_t]
66
+
67
+
68
+def NCCL_CHECK(result: ncclResult_t) -> None:
69
+ if result != 0:
70
+ error_str = _c_ncclGetErrorString(result)
71
+ error_str = error_str.decode("utf-8")
72
+ raise RuntimeError(f"NCCL error: {error_str}")
73
+
74
+
75
# equivalent to c declaration:
76
# ncclResult_t ncclGetVersion(int *version);
77
_c_ncclGetVersion = nccl.ncclGetVersion
@@ -68,8 +81,7 @@ _c_ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
81
82
def ncclGetVersion() -> str:
83
version = ctypes.c_int()
71
- result = _c_ncclGetVersion(ctypes.byref(version))
72
- assert result == 0
84
+ NCCL_CHECK(_c_ncclGetVersion(ctypes.byref(version)))
85
# something like 21903 --> "2.19.3"
86
version_str = str(version.value)
87
major = version_str[0].lstrip("0")
@@ -91,8 +103,7 @@ _c_ncclGetUniqueId.argtypes = [ctypes.POINTER(NcclUniqueId)]
103
104
def ncclGetUniqueId() -> NcclUniqueId:
105
unique_id = NcclUniqueId()
94
- result = _c_ncclGetUniqueId(ctypes.byref(unique_id))
95
- assert result == 0
106
+ NCCL_CHECK(_c_ncclGetUniqueId(ctypes.byref(unique_id)))
107
return unique_id
108
109
@@ -199,66 +210,75 @@ class NCCLCommunicator:
210
211
def __init__(
212
self,
202
- backend=None,
203
- init_method=None,
204
- timeout=datetime.timedelta(seconds=10),
205
- world_size: int = -1,
206
- rank: int = -1,
207
- store=None,
208
- group_name: str = "",
209
- pg_options=None,
210
- local_rank: int = -1,
213
+ group: Optional[ProcessGroup] = None,
214
+ device: Optional[Union[int, str, torch.device]] = None,
215
):
212
- if not dist.is_initialized():
213
- backend = backend or "nccl"
214
- assert backend == 'nccl', (
215
- "only use nccl backend for starting the NCCL communicator")
216
- dist.init_process_group(backend=backend,
217
- init_method=init_method,
218
- timeout=timeout,
219
- world_size=world_size,
220
- rank=rank,
221
- store=store,
222
- group_name=group_name,
223
- pg_options=pg_options)
224
- self.rank = dist.get_rank()
225
- self.world_size = dist.get_world_size()
226
- if local_rank == -1:
227
- local_rank = self.rank
228
- self.local_rank = local_rank
229
- # don't use these args, as they can be -1
230
- # use `self.rank`, `self.local_rank` and `self.world_size` instead
231
- del world_size, rank, local_rank
232
- torch.cuda.set_device(self.local_rank)
216
+ """
217
+ Args:
218
+ group: the process group to work on. If None, it will use the
219
+ default process group.
220
+ device: the device to bind the NCCLCommunicator to. If None,
221
+ it will be bind to f"cuda:{local_rank}".
222
+ It is the caller's responsibility to make sure each communicator
223
+ is bind to a unique device.
224
+ """
225
+ assert dist.is_initialized()
226
+ group = get_cpu_world_group() if group is None else group
227
+ assert dist.get_backend(group) != dist.Backend.NCCL, (
228
+ "NCCLCommunicator should be attached to a non-NCCL group.")
229
+ self.group = group
230
+ self.rank = dist.get_rank(group)
231
+ self.world_size = dist.get_world_size(group)
232
if self.rank == 0:
233
self.unique_id = ncclGetUniqueId()
234
else:
235
self.unique_id = NcclUniqueId()
237
- tensor = torch.ByteTensor(list(self.unique_id.internal)).cuda(
238
- self.local_rank)
239
- dist.broadcast(tensor, src=0)
240
- byte_list = tensor.cpu().tolist()
236
+ tensor = torch.ByteTensor(list(self.unique_id.internal))
237
+ dist.broadcast(tensor, src=0, group=group)
238
+ byte_list = tensor.tolist()
239
for i, byte in enumerate(byte_list):
240
self.unique_id.internal[i] = byte
241
self.comm = ctypes.c_void_p()
244
- result = _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
245
- self.unique_id, self.rank)
246
- assert result == 0
247
- self.stream = torch.cuda.Stream(device=f"cuda:{self.local_rank}")
242
+ if device is None:
243
+ local_rank = get_local_rank()
244
+ device = torch.device(f"cuda:{local_rank}")
245
+ elif isinstance(device, int):
246
+ device = torch.device(f"cuda:{device}")
247
+ elif isinstance(device, str):
248
+ device = torch.device(device)
249
+ # now `device` is a `torch.device` object
250
+ assert isinstance(device, torch.device)
251
+ self.device = device
252
+ # nccl communicator and stream will use this device
253
+ current_device = torch.cuda.current_device()
254
+ try:
255
+ torch.cuda.set_device(device)
256
+ NCCL_CHECK(
257
+ _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
258
+ self.unique_id, self.rank))
259
+ self.stream = torch.cuda.Stream()
260
+ finally:
261
+ torch.cuda.set_device(current_device)
262
263
def all_reduce(self,
264
tensor: torch.Tensor,
265
op: ReduceOp = ReduceOp.SUM,
266
stream=None):
267
+ # nccl communicator created on a specific device
268
+ # will only work on tensors on the same device
269
+ # otherwise it will cause "illegal memory access"
270
+ assert tensor.device == self.device, (
271
+ f"this nccl communicator is created to work on {self.device}, "
272
+ f"but the input tensor is on {tensor.device}")
273
if stream is None:
274
stream = self.stream
255
- result = _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
256
- ctypes.c_void_p(tensor.data_ptr()),
257
- tensor.numel(),
258
- ncclDataTypeEnum.from_torch(tensor.dtype),
259
- ncclRedOpTypeEnum.from_torch(op), self.comm,
260
- ctypes.c_void_p(stream.cuda_stream))
261
- assert result == 0
275
+ NCCL_CHECK(
276
+ _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
277
+ ctypes.c_void_p(tensor.data_ptr()),
278
+ tensor.numel(),
279
+ ncclDataTypeEnum.from_torch(tensor.dtype),
280
+ ncclRedOpTypeEnum.from_torch(op), self.comm,
281
+ ctypes.c_void_p(stream.cuda_stream)))
282
283
def __del__(self):
284
# `dist` module might have been already destroyed
vllm/distributed/device_communicators/pynccl_utils.py
+3
-9
@@ -2,7 +2,7 @@ import contextlib
2
from typing import Optional
3
4
import torch
5
-from torch.distributed import ReduceOp
5
+from torch.distributed import ProcessGroup, ReduceOp
6
7
from vllm.logger import init_logger
8
@@ -37,17 +37,11 @@ def set_pynccl_stream(stream: torch.cuda.Stream):
37
pass
38
39
40
-def init_process_group(world_size: int,
41
- rank: int,
42
- init_method: str,
43
- local_rank: int = -1) -> None:
40
+def init_process_group(group: Optional[ProcessGroup] = None) -> None:
41
assert not is_initialized()
42
global comm
43
logger.info(f"vLLM is using nccl=={ncclGetVersion()}")
47
- comm = NCCLCommunicator(init_method=init_method,
48
- world_size=world_size,
49
- local_rank=local_rank,
50
- rank=rank)
44
+ comm = NCCLCommunicator(group=group)
45
46
47
def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
vllm/distributed/parallel_state.py
+6
@@ -4,6 +4,7 @@
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
import contextlib
7
+import os
8
from typing import Optional
9
10
import torch
@@ -73,6 +74,11 @@ def init_distributed_environment(
74
ranks = list(range(torch.distributed.get_world_size()))
75
_CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
76
backend="gloo")
77
+ # set the local rank
78
+ # local_rank is not available in torch ProcessGroup,
79
+ # see https://github.com/pytorch/pytorch/issues/122816
80
+ if local_rank == -1 and distributed_init_method == "env://":
81
+ local_rank = int(os.environ['LOCAL_RANK'])
82
global _LOCAL_RANK
83
_LOCAL_RANK = local_rank
84
vllm/worker/worker.py
+3
-6
@@ -298,12 +298,9 @@ def init_worker_distributed_environment(
298
elif parallel_config.world_size > 1:
299
# NOTE(woosuk): We don't initialize pynccl process group when world size
300
# is 1.
301
- pynccl_utils.init_process_group(
302
- world_size=parallel_config.world_size,
303
- local_rank=local_rank,
304
- rank=rank,
305
- init_method=distributed_init_method,
306
- )
301
+ # NOTE(kaichao): By default, pynccl will use information inside
302
+ # `parallel_state` for initialization.
303
+ pynccl_utils.init_process_group()
304
305
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
306
parallel_config.pipeline_parallel_size)