A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Distributed] refactor pynccl (#4591)
[Core][Distributed] refactor pynccl to hold multiple communicators (#4591)
youkaichao committed
May 9, 2024 at 19:48 UTC
208b71bcc1b94df1fdd2fc10da3e04c706340188
8 files changed
+467
-434
tests/distributed/test_pynccl.py
+42
-36
@@ -1,15 +1,15 @@
1
import multiprocessing
2
+import os
3
4
import pytest
5
import torch
6
6
-import vllm.distributed.device_communicators.pynccl_utils as pynccl_utils
7
-from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
8
-from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
9
- ncclGetUniqueId)
10
-from vllm.distributed.parallel_state import (
11
- ensure_model_parallel_initialized, get_tensor_model_parallel_cpu_group,
12
- init_distributed_environment, with_pynccl_for_all_reduce)
7
+from vllm.distributed.communication_op import ( # noqa
8
+ graph_capture_mode, tensor_model_parallel_all_reduce)
9
+from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
10
+from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
11
+from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
12
+ init_distributed_environment)
13
from vllm.utils import update_environment_variables
14
15
@@ -41,6 +41,9 @@ def worker_fn_wrapper(fn):
41
# and update the environment variables in the function
42
def wrapped_fn(env):
43
update_environment_variables(env)
44
+ local_rank = os.environ['LOCAL_RANK']
45
+ device = torch.device(f"cuda:{local_rank}")
46
+ torch.cuda.set_device(device)
47
init_distributed_environment()
48
fn()
49
@@ -49,11 +52,13 @@ def worker_fn_wrapper(fn):
52
53
@worker_fn_wrapper
54
def worker_fn():
52
- comm = NCCLCommunicator()
53
- tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
54
- comm.all_reduce(tensor)
55
+ pynccl_comm = PyNcclCommunicator()
56
+ tensor = torch.ones(16, 1024, 1024,
57
+ dtype=torch.float32).cuda(pynccl_comm.rank)
58
+ with pynccl_comm.change_state(enable=True):
59
+ pynccl_comm.all_reduce(tensor)
60
result = tensor.mean().cpu().item()
56
- assert result == comm.world_size
61
+ assert result == pynccl_comm.world_size
62
63
64
@pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -70,37 +75,35 @@ def multiple_tp_worker_fn():
75
torch.distributed.new_group(ranks=[2, 3], backend="gloo")
76
]
77
group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
73
- comm = NCCLCommunicator(group=group, device=device)
78
+ pynccl_comm = PyNcclCommunicator(group=group, device=device)
79
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
75
- # two groups can communicate independently
76
- if torch.distributed.get_rank() in [0, 1]:
77
- comm.all_reduce(tensor)
78
- comm.all_reduce(tensor)
79
- result = tensor.mean().cpu().item()
80
- assert result == 4
81
- else:
82
- comm.all_reduce(tensor)
83
- result = tensor.mean().cpu().item()
84
- assert result == 2
80
+ with pynccl_comm.change_state(enable=True):
81
+ # two groups can communicate independently
82
+ if torch.distributed.get_rank() in [0, 1]:
83
+ pynccl_comm.all_reduce(tensor)
84
+ pynccl_comm.all_reduce(tensor)
85
+ result = tensor.mean().cpu().item()
86
+ assert result == 4
87
+ else:
88
+ pynccl_comm.all_reduce(tensor)
89
+ result = tensor.mean().cpu().item()
90
+ assert result == 2
91
92
93
@pytest.mark.skipif(torch.cuda.device_count() < 4,
94
reason="Need at least 4 GPUs to run the test.")
95
def test_pynccl_multiple_tp():
96
# this tests pynccl for multiple tp groups, in a standalone way
91
- # i.e. call `comm.all_reduce` directly
97
+ # i.e. call `pynccl_comm.all_reduce` directly
98
distributed_run(multiple_tp_worker_fn, 4)
99
100
101
@worker_fn_wrapper
102
def multiple_tp_with_vllm_worker_fn():
103
device = torch.device(f"cuda:{torch.distributed.get_rank()}")
98
- torch.cuda.set_device(torch.distributed.get_rank())
104
ensure_model_parallel_initialized(2, 2)
100
- pynccl_utils.init_process_group(
101
- group=get_tensor_model_parallel_cpu_group())
105
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
103
- with with_pynccl_for_all_reduce():
106
+ with graph_capture_mode():
107
# two tp groups can communicate independently
108
if torch.distributed.get_rank() in [0, 1]:
109
tensor = tensor_model_parallel_all_reduce(tensor)
@@ -125,19 +128,21 @@ def test_pynccl_multiple_tp_with_vllm():
128
def worker_fn_with_cudagraph():
129
with torch.no_grad():
130
graph = torch.cuda.CUDAGraph()
128
- comm = NCCLCommunicator()
131
+ pynccl_comm = PyNcclCommunicator()
132
# run something in the default stream to initialize torch engine
130
- a = torch.ones((4, 4), device=f'cuda:{comm.rank}')
133
+ a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
134
torch.cuda.synchronize()
132
- with torch.cuda.graph(graph, stream=comm.stream):
135
+ with torch.cuda.graph(
136
+ graph, stream=pynccl_comm.stream), pynccl_comm.change_state(
137
+ enable=True):
138
# operation during the graph capture is recorded but not executed
139
# see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#creating-a-graph-using-stream-capture # noqa
135
- comm.all_reduce(a)
136
- comm.stream.synchronize()
137
- assert a.mean().cpu().item() == comm.world_size**0
140
+ pynccl_comm.all_reduce(a)
141
+ pynccl_comm.stream.synchronize()
142
+ assert a.mean().cpu().item() == pynccl_comm.world_size**0
143
graph.replay()
139
- comm.stream.synchronize()
140
- assert a.mean().cpu().item() == comm.world_size**1
144
+ pynccl_comm.stream.synchronize()
145
+ assert a.mean().cpu().item() == pynccl_comm.world_size**1
146
147
148
@pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -147,7 +152,8 @@ def test_pynccl_with_cudagraph():
152
153
154
def test_ncclGetUniqueId():
150
- unique_id = ncclGetUniqueId()
155
+ lib = NCCLLibrary()
156
+ unique_id = lib.ncclGetUniqueId()
157
# `list(unique_id.internal)` is something like this:
158
# [34, -16, 23, 83, 109, -19, 59, 95, 2, 0, -86, 55, 10, -128, 0, 29, 0,
159
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
vllm/distributed/communication_op.py
+24
-4
@@ -1,4 +1,5 @@
1
from collections import namedtuple
2
+from contextlib import contextmanager
3
from typing import Any, Dict, List, Optional, Tuple, Union
4
5
import torch
@@ -8,7 +9,26 @@ from .parallel_state import (get_cpu_world_group,
9
get_tensor_model_parallel_group,
10
get_tensor_model_parallel_rank,
11
get_tensor_model_parallel_world_size,
11
- is_pynccl_enabled_for_all_reduce)
12
+ get_tp_pynccl_communicator)
13
+
14
+
15
+@contextmanager
16
+def graph_capture_mode():
17
+ # In graph capture, we have to be very careful about the collective
18
+ # operations. The current status is:
19
+ # allreduce \ Mode | Eager | Graph |
20
+ # --------------------------------------------
21
+ # custom allreduce | enabled | enabled |
22
+ # PyNccl | disabled| enabled |
23
+ # torch.distributed | enabled | disabled|
24
+ #
25
+ # Note that custom allreduce will have a runtime check, if the tensor size
26
+ # is too large, it will fallback to the next available option.
27
+ pynccl_comm = get_tp_pynccl_communicator()
28
+ assert pynccl_comm is not None
29
+ with pynccl_comm.change_state(enable=True,
30
+ stream=torch.cuda.current_stream()):
31
+ yield
32
33
34
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -23,7 +43,6 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
43
TLDR: always assume this function modifies its input, but use the return
44
value as the output.
45
"""
26
- from vllm.distributed.device_communicators import pynccl_utils
46
from vllm.distributed.device_communicators.custom_all_reduce import (
47
custom_all_reduce)
48
@@ -33,8 +52,9 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
52
out = custom_all_reduce(input_)
53
if out is not None:
54
return out
36
- if is_pynccl_enabled_for_all_reduce():
37
- pynccl_utils.all_reduce(input_)
55
+ pynccl_comm = get_tp_pynccl_communicator()
56
+ if (pynccl_comm is not None and not pynccl_comm.disabled):
57
+ pynccl_comm.all_reduce(input_)
58
else:
59
torch.distributed.all_reduce(input_,
60
group=get_tensor_model_parallel_group())
vllm/distributed/device_communicators/pynccl.py
+78
-216
@@ -1,26 +1,4 @@
1
-# This file is a pure Python wrapper for the NCCL library.
2
-# The main purpose is to use NCCL combined with CUDA graph.
3
-# Before writing this script, we tried the following approach:
4
-# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself
5
-# often gets stuck when initializing the NCCL communicator.
6
-# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce`
7
-# contains many other potential cuda APIs, that are not allowed during
8
-# capturing the CUDA graph. For further details, please check
9
-# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ .
10
-#
11
-# Another rejected idea is to write a C/C++ binding for NCCL. It is usually
12
-# doable, but we often encounter issues related with nccl versions, and need
13
-# to switch between different versions of NCCL. See
14
-# https://github.com/NVIDIA/nccl/issues/1234 for more details.
15
-# A C/C++ binding is not flexible enough to handle this. It requires
16
-# recompilation of the code every time we want to switch between different
17
-# versions. This current implementation, with a **pure** Python wrapper, is
18
-# more flexible. We can easily switch between different versions of NCCL by
19
-# changing the environment variable `VLLM_NCCL_SO_PATH`, or the `so_file`
20
-# variable in the code.
21
-
22
-import ctypes
23
-import platform
1
+from contextlib import contextmanager
2
from typing import Optional, Union
3
4
# ===================== import region =====================
@@ -28,217 +6,70 @@ import torch
6
import torch.distributed as dist
7
from torch.distributed import ProcessGroup, ReduceOp
8
9
+from vllm.distributed.device_communicators.pynccl_wrapper import (
10
+ NCCLLibrary, buffer_type, cudaStream_t, ncclComm_t, ncclDataTypeEnum,
11
+ ncclRedOpTypeEnum, ncclUniqueId)
12
from vllm.distributed.parallel_state import get_cpu_world_group, get_local_rank
13
from vllm.logger import init_logger
33
-from vllm.utils import find_nccl_library, nccl_integrity_check
14
15
logger = init_logger(__name__)
16
37
-so_file = find_nccl_library()
38
-
39
-try:
40
- # load the library in another process.
41
- # if it core dumps, it will not crash the current process
42
- nccl_integrity_check(so_file)
43
- nccl = ctypes.CDLL(so_file)
44
-except Exception as e:
45
- logger.error(
46
- "Failed to load NCCL library from %s ."
47
- "It is expected if you are not running on NVIDIA/AMD GPUs."
48
- "Otherwise, the nccl library might not exist, be corrupted "
49
- "or it does not support the current platform %s."
50
- "One solution is to download libnccl2 version 2.18 from "
51
- "https://developer.download.nvidia.com/compute/cuda/repos/ "
52
- "and extract the libnccl.so.2 file. If you already have the "
53
- "library, please set the environment variable VLLM_NCCL_SO_PATH"
54
- " to point to the correct nccl library path.", so_file,
55
- platform.platform())
56
- raise e
57
-
58
-# === export types and functions from nccl to Python ===
59
-# for the original nccl definition, please check
60
-# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in
61
-
62
-ncclResult_t = ctypes.c_int
63
-
64
-_c_ncclGetErrorString = nccl.ncclGetErrorString
65
-_c_ncclGetErrorString.restype = ctypes.c_char_p
66
-_c_ncclGetErrorString.argtypes = [ncclResult_t]
67
-
68
-
69
-def NCCL_CHECK(result: ncclResult_t) -> None:
70
- if result != 0:
71
- error_str = _c_ncclGetErrorString(result)
72
- error_str = error_str.decode("utf-8")
73
- raise RuntimeError(f"NCCL error: {error_str}")
74
-
75
-
76
-# equivalent to c declaration:
77
-# ncclResult_t ncclGetVersion(int *version);
78
-_c_ncclGetVersion = nccl.ncclGetVersion
79
-_c_ncclGetVersion.restype = ctypes.c_int
80
-_c_ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
81
-
82
-
83
-def ncclGetVersion() -> str:
84
- version = ctypes.c_int()
85
- NCCL_CHECK(_c_ncclGetVersion(ctypes.byref(version)))
86
- # something like 21903 --> "2.19.3"
87
- version_str = str(version.value)
88
- major = version_str[0].lstrip("0")
89
- minor = version_str[1:3].lstrip("0")
90
- patch = version_str[3:].lstrip("0")
91
- return f"{major}.{minor}.{patch}"
92
-
93
-
94
-class NcclUniqueId(ctypes.Structure):
95
- _fields_ = [("internal", ctypes.c_byte * 128)]
96
-
97
-
98
-# equivalent to c declaration:
99
-# ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId);
100
-_c_ncclGetUniqueId = nccl.ncclGetUniqueId
101
-_c_ncclGetUniqueId.restype = ctypes.c_int
102
-_c_ncclGetUniqueId.argtypes = [ctypes.POINTER(NcclUniqueId)]
103
-
104
-
105
-def ncclGetUniqueId() -> NcclUniqueId:
106
- unique_id = NcclUniqueId()
107
- NCCL_CHECK(_c_ncclGetUniqueId(ctypes.byref(unique_id)))
108
- return unique_id
109
-
110
-
111
-# equivalent to c declaration:
112
-# ncclResult_t ncclCommInitRank(
113
-# ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank);
114
-# note that ncclComm_t is a pointer type, so the first argument
115
-# is a pointer to a pointer
116
-_c_ncclCommInitRank = nccl.ncclCommInitRank
117
-_c_ncclCommInitRank.restype = ctypes.c_int
118
-_c_ncclCommInitRank.argtypes = [
119
- ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, NcclUniqueId, ctypes.c_int
120
-]
121
-
122
-ncclDataType_t = ctypes.c_int
123
-
124
-
125
-class ncclDataTypeEnum:
126
- ncclInt8 = 0
127
- ncclChar = 0
128
- ncclUint8 = 1
129
- ncclInt32 = 2
130
- ncclInt = 2
131
- ncclUint32 = 3
132
- ncclInt64 = 4
133
- ncclUint64 = 5
134
- ncclFloat16 = 6
135
- ncclHalf = 6
136
- ncclFloat32 = 7
137
- ncclFloat = 7
138
- ncclFloat64 = 8
139
- ncclDouble = 8
140
- ncclBfloat16 = 9
141
- ncclNumTypes = 10
17
143
- @classmethod
144
- def from_torch(cls, dtype: torch.dtype) -> int:
145
- if dtype == torch.int8:
146
- return cls.ncclInt8
147
- if dtype == torch.uint8:
148
- return cls.ncclUint8
149
- if dtype == torch.int32:
150
- return cls.ncclInt32
151
- if dtype == torch.int64:
152
- return cls.ncclInt64
153
- if dtype == torch.float16:
154
- return cls.ncclFloat16
155
- if dtype == torch.float32:
156
- return cls.ncclFloat32
157
- if dtype == torch.float64:
158
- return cls.ncclFloat64
159
- if dtype == torch.bfloat16:
160
- return cls.ncclBfloat16
161
- raise ValueError(f"Unsupported dtype: {dtype}")
162
-
163
-
164
-ncclRedOp_t = ctypes.c_int
165
-
166
-
167
-class ncclRedOpTypeEnum:
168
- ncclSum = 0
169
- ncclProd = 1
170
- ncclMax = 2
171
- ncclMin = 3
172
- ncclAvg = 4
173
- ncclNumOps = 5
174
-
175
- @classmethod
176
- def from_torch(cls, op: ReduceOp) -> int:
177
- if op == ReduceOp.SUM:
178
- return cls.ncclSum
179
- if op == ReduceOp.PRODUCT:
180
- return cls.ncclProd
181
- if op == ReduceOp.MAX:
182
- return cls.ncclMax
183
- if op == ReduceOp.MIN:
184
- return cls.ncclMin
185
- if op == ReduceOp.AVG:
186
- return cls.ncclAvg
187
- raise ValueError(f"Unsupported op: {op}")
188
-
189
-
190
-# equivalent to c declaration:
191
-# ncclResult_t ncclAllReduce(
192
-# const void* sendbuff, void* recvbuff, size_t count,
193
-# ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
194
-# udaStream_t stream);
195
-# note that cudaStream_t is a pointer type, so the last argument is a pointer
196
-_c_ncclAllReduce = nccl.ncclAllReduce
197
-_c_ncclAllReduce.restype = ctypes.c_int
198
-_c_ncclAllReduce.argtypes = [
199
- ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclRedOp_t,
200
- ncclDataType_t, ctypes.c_void_p, ctypes.c_void_p
201
-]
202
-
203
-# be cautious! this is a collective call, it will block until all
204
-# processes in the communicator have called this function.
205
-# because Python object destruction can happen in random order,
206
-# it is better not to call it at all.
207
-# equivalent to c declaration:
208
-# ncclResult_t ncclCommDestroy(ncclComm_t comm);
209
-_c_ncclCommDestroy = nccl.ncclCommDestroy
210
-_c_ncclCommDestroy.restype = ctypes.c_int
211
-_c_ncclCommDestroy.argtypes = [ctypes.c_void_p]
212
-
213
-
214
-class NCCLCommunicator:
18
+class PyNcclCommunicator:
19
20
def __init__(
21
self,
22
group: Optional[ProcessGroup] = None,
23
device: Optional[Union[int, str, torch.device]] = None,
24
+ library_path: Optional[str] = None,
25
):
26
"""
27
Args:
28
group: the process group to work on. If None, it will use the
29
default process group.
225
- device: the device to bind the NCCLCommunicator to. If None,
30
+ device: the device to bind the PyNcclCommunicator to. If None,
31
it will be bind to f"cuda:{local_rank}".
32
+ library_path: the path to the NCCL library. If None, it will
33
+ use the default library path.
34
It is the caller's responsibility to make sure each communicator
35
is bind to a unique device.
36
"""
37
assert dist.is_initialized()
38
group = get_cpu_world_group() if group is None else group
39
assert dist.get_backend(group) != dist.Backend.NCCL, (
233
- "NCCLCommunicator should be attached to a non-NCCL group.")
40
+ "PyNcclCommunicator should be attached to a non-NCCL group.")
41
self.group = group
42
# note: this rank is the rank in the group
43
self.rank = dist.get_rank(group)
44
self.world_size = dist.get_world_size(group)
45
+
46
+ # if world_size == 1, no need to create communicator
47
+ if self.world_size == 1:
48
+ self.available = False
49
+ self.disabled = True
50
+ self.stream = None
51
+ return
52
+ try:
53
+ self.nccl = NCCLLibrary(library_path)
54
+ except Exception:
55
+ # disable because of missing NCCL library
56
+ # e.g. in a non-GPU environment
57
+ self.available = False
58
+ self.disabled = True
59
+ self.stream = None
60
+ return
61
+
62
+ self.available = True
63
+ self.disabled = False
64
+
65
+ logger.info("vLLM is using nccl==%s", self.nccl.ncclGetVersion())
66
+
67
if self.rank == 0:
239
- self.unique_id = ncclGetUniqueId()
68
+ # get the unique id from NCCL
69
+ self.unique_id = self.nccl.ncclGetUniqueId()
70
else:
241
- self.unique_id = NcclUniqueId()
71
+ # construct an empty unique id
72
+ self.unique_id = ncclUniqueId()
73
tensor = torch.ByteTensor(list(self.unique_id.internal))
74
ranks = dist.get_process_group_ranks(group)
75
# arg `src` in `broadcast` is the global rank
@@ -246,7 +77,6 @@ class NCCLCommunicator:
77
byte_list = tensor.tolist()
78
for i, byte in enumerate(byte_list):
79
self.unique_id.internal[i] = byte
249
- self.comm = ctypes.c_void_p()
80
if device is None:
81
local_rank = get_local_rank()
82
device = torch.device(f"cuda:{local_rank}")
@@ -261,15 +91,25 @@ class NCCLCommunicator:
91
# `torch.cuda.device` is a context manager that changes the
92
# current cuda device to the specified one
93
with torch.cuda.device(device):
264
- NCCL_CHECK(
265
- _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
266
- self.unique_id, self.rank))
94
+ self.comm: ncclComm_t = self.nccl.ncclCommInitRank(
95
+ self.world_size, self.unique_id, self.rank)
96
self.stream = torch.cuda.Stream()
97
98
+ # A small all_reduce for warmup.
99
+ self.all_reduce(torch.zeros(1, device=device))
100
+ self.stream.synchronize()
101
+
102
+ # by default it is disabled, e.g. in profiling models and prefill phase.
103
+ # to use it, use under `with obj.change_state(enable=True)`, usually
104
+ # when we are using CUDA graph.
105
+ self.disabled = True
106
+
107
def all_reduce(self,
108
tensor: torch.Tensor,
109
op: ReduceOp = ReduceOp.SUM,
110
stream=None):
111
+ if self.disabled:
112
+ return
113
# nccl communicator created on a specific device
114
# will only work on tensors on the same device
115
# otherwise it will cause "illegal memory access"
@@ -278,10 +118,32 @@ class NCCLCommunicator:
118
f"but the input tensor is on {tensor.device}")
119
if stream is None:
120
stream = self.stream
281
- NCCL_CHECK(
282
- _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
283
- ctypes.c_void_p(tensor.data_ptr()),
284
- tensor.numel(),
285
- ncclDataTypeEnum.from_torch(tensor.dtype),
286
- ncclRedOpTypeEnum.from_torch(op), self.comm,
287
- ctypes.c_void_p(stream.cuda_stream)))
121
+ self.nccl.ncclAllReduce(buffer_type(tensor.data_ptr()),
122
+ buffer_type(tensor.data_ptr()), tensor.numel(),
123
+ ncclDataTypeEnum.from_torch(tensor.dtype),
124
+ ncclRedOpTypeEnum.from_torch(op), self.comm,
125
+ cudaStream_t(stream.cuda_stream))
126
+
127
+ @contextmanager
128
+ def change_state(self,
129
+ enable: Optional[bool] = None,
130
+ stream: Optional[torch.cuda.Stream] = None):
131
+ """
132
+ A context manager to change the state of the communicator.
133
+ """
134
+ if enable is None:
135
+ # guess a default value when not specified
136
+ enable = self.available
137
+
138
+ if stream is None:
139
+ stream = self.stream
140
+
141
+ old_disable = self.disabled
142
+ old_stream = self.stream
143
+
144
+ self.stream = stream
145
+ self.disabled = not enable
146
+ yield
147
+
148
+ self.disabled = old_disable
149
+ self.stream = old_stream
vllm/distributed/device_communicators/pynccl_utils.py
deleted
-66
@@ -1,66 +0,0 @@
1
-import contextlib
2
-from typing import Optional
3
-
4
-import torch
5
-from torch.distributed import ProcessGroup, ReduceOp
6
-
7
-from vllm.logger import init_logger
8
-
9
-logger = init_logger(__name__)
10
-
11
-try:
12
- from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
13
- ncclGetVersion)
14
-except Exception as e:
15
- # in non-NVIDIA environments, we can't import the nccl module
16
- # e.g. when running on machines with AMD GPUs
17
- logger.info("Failed to import NCCL library: %s", e)
18
- logger.info("It is expected if you are not running on NVIDIA GPUs.")
19
- pass
20
-
21
-comm: Optional["NCCLCommunicator"] = None
22
-
23
-
24
-def is_initialized() -> bool:
25
- """Returns whether the NCCL backend is initialized."""
26
- return comm is not None
27
-
28
-
29
-@contextlib.contextmanager
30
-def set_pynccl_stream(stream: torch.cuda.Stream):
31
- """Set the cuda stream for communication"""
32
- try:
33
- assert comm is not None
34
- comm.stream = stream
35
- yield
36
- finally:
37
- pass
38
-
39
-
40
-def init_process_group(group: Optional[ProcessGroup] = None) -> None:
41
- assert not is_initialized()
42
- global comm
43
- logger.info("vLLM is using nccl==%s", ncclGetVersion())
44
- comm = NCCLCommunicator(group=group)
45
-
46
-
47
-def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
48
- """All-reduces the input tensor across the process group."""
49
- assert input_.is_cuda, f"{input_} should be a cuda tensor"
50
- assert comm is not None
51
- comm.all_reduce(input_, op)
52
-
53
-
54
-def destroy_process_group() -> None:
55
- global comm
56
- comm = None
57
-
58
-
59
-def get_world_size() -> int:
60
- """Returns the world size."""
61
- assert comm is not None
62
- return comm.world_size
63
-
64
-
65
-def get_nccl_backend() -> Optional["NCCLCommunicator"]:
66
- return comm
vllm/distributed/device_communicators/pynccl_wrapper.py
new
+258
@@ -0,0 +1,258 @@
1
+# This file is a pure Python wrapper for the NCCL library.
2
+# The main purpose is to use NCCL combined with CUDA graph.
3
+# Before writing this script, we tried the following approach:
4
+# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself
5
+# often gets stuck when initializing the NCCL communicator.
6
+# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce`
7
+# contains many other potential cuda APIs, that are not allowed during
8
+# capturing the CUDA graph. For further details, please check
9
+# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ .
10
+#
11
+# Another rejected idea is to write a C/C++ binding for NCCL. It is usually
12
+# doable, but we often encounter issues related with nccl versions, and need
13
+# to switch between different versions of NCCL. See
14
+# https://github.com/NVIDIA/nccl/issues/1234 for more details.
15
+# A C/C++ binding is not flexible enough to handle this. It requires
16
+# recompilation of the code every time we want to switch between different
17
+# versions. This current implementation, with a **pure** Python wrapper, is
18
+# more flexible. We can easily switch between different versions of NCCL by
19
+# changing the environment variable `VLLM_NCCL_SO_PATH`, or the `so_file`
20
+# variable in the code.
21
+
22
+import ctypes
23
+import platform
24
+from dataclasses import dataclass
25
+from typing import Any, Dict, List, Optional
26
+
27
+import torch
28
+from torch.distributed import ReduceOp
29
+
30
+from vllm.logger import init_logger
31
+from vllm.utils import find_nccl_library, nccl_integrity_check
32
+
33
+logger = init_logger(__name__)
34
+
35
+# === export types and functions from nccl to Python ===
36
+# for the original nccl definition, please check
37
+# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in
38
+
39
+ncclResult_t = ctypes.c_int
40
+ncclComm_t = ctypes.c_void_p
41
+
42
+
43
+class ncclUniqueId(ctypes.Structure):
44
+ _fields_ = [("internal", ctypes.c_byte * 128)]
45
+
46
+
47
+cudaStream_t = ctypes.c_void_p
48
+buffer_type = ctypes.c_void_p
49
+
50
+ncclDataType_t = ctypes.c_int
51
+
52
+
53
+class ncclDataTypeEnum:
54
+ ncclInt8 = 0
55
+ ncclChar = 0
56
+ ncclUint8 = 1
57
+ ncclInt32 = 2
58
+ ncclInt = 2
59
+ ncclUint32 = 3
60
+ ncclInt64 = 4
61
+ ncclUint64 = 5
62
+ ncclFloat16 = 6
63
+ ncclHalf = 6
64
+ ncclFloat32 = 7
65
+ ncclFloat = 7
66
+ ncclFloat64 = 8
67
+ ncclDouble = 8
68
+ ncclBfloat16 = 9
69
+ ncclNumTypes = 10
70
+
71
+ @classmethod
72
+ def from_torch(cls, dtype: torch.dtype) -> int:
73
+ if dtype == torch.int8:
74
+ return cls.ncclInt8
75
+ if dtype == torch.uint8:
76
+ return cls.ncclUint8
77
+ if dtype == torch.int32:
78
+ return cls.ncclInt32
79
+ if dtype == torch.int64:
80
+ return cls.ncclInt64
81
+ if dtype == torch.float16:
82
+ return cls.ncclFloat16
83
+ if dtype == torch.float32:
84
+ return cls.ncclFloat32
85
+ if dtype == torch.float64:
86
+ return cls.ncclFloat64
87
+ if dtype == torch.bfloat16:
88
+ return cls.ncclBfloat16
89
+ raise ValueError(f"Unsupported dtype: {dtype}")
90
+
91
+
92
+ncclRedOp_t = ctypes.c_int
93
+
94
+
95
+class ncclRedOpTypeEnum:
96
+ ncclSum = 0
97
+ ncclProd = 1
98
+ ncclMax = 2
99
+ ncclMin = 3
100
+ ncclAvg = 4
101
+ ncclNumOps = 5
102
+
103
+ @classmethod
104
+ def from_torch(cls, op: ReduceOp) -> int:
105
+ if op == ReduceOp.SUM:
106
+ return cls.ncclSum
107
+ if op == ReduceOp.PRODUCT:
108
+ return cls.ncclProd
109
+ if op == ReduceOp.MAX:
110
+ return cls.ncclMax
111
+ if op == ReduceOp.MIN:
112
+ return cls.ncclMin
113
+ if op == ReduceOp.AVG:
114
+ return cls.ncclAvg
115
+ raise ValueError(f"Unsupported op: {op}")
116
+
117
+
118
+@dataclass
119
+class Function:
120
+ name: str
121
+ restype: Any
122
+ argtypes: List[Any]
123
+
124
+
125
+class NCCLLibrary:
126
+ exported_functions = [
127
+ # const char* ncclGetErrorString(ncclResult_t result)
128
+ Function("ncclGetErrorString", ctypes.c_char_p, [ncclResult_t]),
129
+ # ncclResult_t ncclGetVersion(int *version);
130
+ Function("ncclGetVersion", ncclResult_t,
131
+ [ctypes.POINTER(ctypes.c_int)]),
132
+ # ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId);
133
+ Function("ncclGetUniqueId", ncclResult_t,
134
+ [ctypes.POINTER(ncclUniqueId)]),
135
+ # ncclResult_t ncclCommInitRank(
136
+ # ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank);
137
+ # note that ncclComm_t is a pointer type, so the first argument
138
+ # is a pointer to a pointer
139
+ Function("ncclCommInitRank", ncclResult_t, [
140
+ ctypes.POINTER(ncclComm_t), ctypes.c_int, ncclUniqueId,
141
+ ctypes.c_int
142
+ ]),
143
+ # ncclResult_t ncclAllReduce(
144
+ # const void* sendbuff, void* recvbuff, size_t count,
145
+ # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
146
+ # cudaStream_t stream);
147
+ # note that cudaStream_t is a pointer type, so the last argument
148
+ # is a pointer
149
+ Function("ncclAllReduce", ncclResult_t, [
150
+ buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
151
+ ncclRedOp_t, ncclComm_t, cudaStream_t
152
+ ]),
153
+
154
+ # be cautious! this is a collective call, it will block until all
155
+ # processes in the communicator have called this function.
156
+ # because Python object destruction can happen in random order,
157
+ # it is better not to call it at all.
158
+ # ncclResult_t ncclCommDestroy(ncclComm_t comm);
159
+ Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]),
160
+ ]
161
+
162
+ # class attribute to store the mapping from the path to the library
163
+ # to avoid loading the same library multiple times
164
+ path_to_library_cache: Dict[str, Any] = {}
165
+
166
+ # class attribute to store the mapping from library path
167
+ # to the corresponding dictionary
168
+ path_to_dict_mapping: Dict[str, Dict[str, Any]] = {}
169
+
170
+ def __init__(self, so_file: Optional[str] = None):
171
+
172
+ so_file = so_file or find_nccl_library()
173
+
174
+ try:
175
+ # load the library in another process.
176
+ # if it core dumps, it will not crash the current process
177
+ nccl_integrity_check(so_file)
178
+ except Exception as e:
179
+ logger.error(
180
+ "Failed to load NCCL library from %s ."
181
+ "It is expected if you are not running on NVIDIA/AMD GPUs."
182
+ "Otherwise, the nccl library might not exist, be corrupted "
183
+ "or it does not support the current platform %s."
184
+ "One solution is to download libnccl2 version 2.18 from "
185
+ "https://developer.download.nvidia.com/compute/cuda/repos/ "
186
+ "and extract the libnccl.so.2 file. If you already have the "
187
+ "library, please set the environment variable VLLM_NCCL_SO_PATH"
188
+ " to point to the correct nccl library path.", so_file,
189
+ platform.platform())
190
+ raise e
191
+
192
+ if so_file not in NCCLLibrary.path_to_dict_mapping:
193
+ lib = ctypes.CDLL(so_file)
194
+ NCCLLibrary.path_to_library_cache[so_file] = lib
195
+ self.lib = NCCLLibrary.path_to_library_cache[so_file]
196
+
197
+ if so_file not in NCCLLibrary.path_to_dict_mapping:
198
+ _funcs = {}
199
+ for func in NCCLLibrary.exported_functions:
200
+ f = getattr(self.lib, func.name)
201
+ f.restype = func.restype
202
+ f.argtypes = func.argtypes
203
+ _funcs[func.name] = f
204
+ NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
205
+ self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
206
+
207
+ def ncclGetErrorString(self, result: ncclResult_t) -> str:
208
+ return self._funcs["ncclGetErrorString"](result).decode("utf-8")
209
+
210
+ def NCCL_CHECK(self, result: ncclResult_t) -> None:
211
+ if result != 0:
212
+ error_str = self.ncclGetErrorString(result)
213
+ raise RuntimeError(f"NCCL error: {error_str}")
214
+
215
+ def ncclGetVersion(self) -> str:
216
+ version = ctypes.c_int()
217
+ self.NCCL_CHECK(self._funcs["ncclGetVersion"](ctypes.byref(version)))
218
+ version_str = str(version.value)
219
+ # something like 21903 --> "2.19.3"
220
+ major = version_str[0].lstrip("0")
221
+ minor = version_str[1:3].lstrip("0")
222
+ patch = version_str[3:].lstrip("0")
223
+ return f"{major}.{minor}.{patch}"
224
+
225
+ def ncclGetUniqueId(self) -> ncclUniqueId:
226
+ unique_id = ncclUniqueId()
227
+ self.NCCL_CHECK(self._funcs["ncclGetUniqueId"](
228
+ ctypes.byref(unique_id)))
229
+ return unique_id
230
+
231
+ def ncclCommInitRank(self, world_size: int, unique_id: ncclUniqueId,
232
+ rank: int) -> ncclComm_t:
233
+ comm = ncclComm_t()
234
+ self.NCCL_CHECK(self._funcs["ncclCommInitRank"](ctypes.byref(comm),
235
+ world_size, unique_id,
236
+ rank))
237
+ return comm
238
+
239
+ def ncclAllReduce(self, sendbuff: buffer_type, recvbuff: buffer_type,
240
+ count: int, datatype: int, op: int, comm: ncclComm_t,
241
+ stream: cudaStream_t) -> None:
242
+ # `datatype` actually should be `ncclDataType_t`
243
+ # and `op` should be `ncclRedOp_t`
244
+ # both are aliases of `ctypes.c_int`
245
+ # when we pass int to a function, it will be converted to `ctypes.c_int`
246
+ # by ctypes automatically
247
+ self.NCCL_CHECK(self._funcs["ncclAllReduce"](sendbuff, recvbuff, count,
248
+ datatype, op, comm,
249
+ stream))
250
+
251
+ def ncclCommDestroy(self, comm: ncclComm_t) -> None:
252
+ self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
253
+
254
+
255
+__all__ = [
256
+ "NCCLLibrary", "ncclDataTypeEnum", "ncclRedOpTypeEnum", "ncclUniqueId",
257
+ "ncclComm_t", "cudaStream_t", "buffer_type"
258
+]
vllm/distributed/parallel_state.py
+60
-71
@@ -3,10 +3,10 @@
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
-import contextlib
7
-from typing import Optional
6
+from typing import List, Optional
7
8
import torch
9
+from torch.distributed import ProcessGroup
10
11
import vllm.envs as envs
12
from vllm.logger import init_logger
@@ -14,10 +14,11 @@ from vllm.logger import init_logger
14
logger = init_logger(__name__)
15
16
# Tensor model parallel group that the current rank belongs to.
17
-_TP_DEVICE_GROUP = None
18
-_TP_CPU_GROUP = None
17
+_TP_DEVICE_GROUP: Optional[ProcessGroup] = None
18
+_TP_CPU_GROUP: Optional[ProcessGroup] = None
19
+_TP_PYNCCL_COMMUNICATOR = None
20
# Pipeline model parallel group that the current rank belongs to.
20
-_PIPELINE_MODEL_PARALLEL_GROUP = None
21
+_PP_DEVICE_GROUP: Optional[ProcessGroup] = None
22
23
# when people blindly call `torch.distributed.all_reduce` etc,
24
# it will use this group. It is initialized with the `backend`
@@ -41,11 +42,16 @@ _CPU_WORLD_GROUP = None
42
43
# A list of global ranks for each pipeline group to ease calculation of the
44
# source rank when broadcasting from the first or last pipeline stage.
44
-_PIPELINE_GLOBAL_RANKS = None
45
+_PP_GLOBAL_RANKS: Optional[List[int]] = None
46
47
_LOCAL_RANK = -1
48
49
50
+def get_tp_pynccl_communicator():
51
+ global _TP_PYNCCL_COMMUNICATOR
52
+ return _TP_PYNCCL_COMMUNICATOR
53
+
54
+
55
def get_local_rank():
56
global _LOCAL_RANK
57
return _LOCAL_RANK
@@ -80,10 +86,20 @@ def init_distributed_environment(
86
# set the local rank
87
# local_rank is not available in torch ProcessGroup,
88
# see https://github.com/pytorch/pytorch/issues/122816
83
- if local_rank == -1 and distributed_init_method == "env://":
84
- local_rank = envs.LOCAL_RANK
89
+ if local_rank == -1:
90
+ # local rank not set, this usually happens in single-node
91
+ # setting, where we can use rank as local rank
92
+ if distributed_init_method == "env://":
93
+ local_rank = envs.LOCAL_RANK
94
+ else:
95
+ local_rank = rank
96
global _LOCAL_RANK
97
_LOCAL_RANK = local_rank
98
+ # A small all_reduce for warmup.
99
+ data = torch.zeros(1)
100
+ if torch.cuda.is_available():
101
+ data = data.to(device=f"cuda:{local_rank}")
102
+ torch.distributed.all_reduce(data)
103
104
105
def initialize_model_parallel(
@@ -133,29 +149,36 @@ def initialize_model_parallel(
149
rank = torch.distributed.get_rank()
150
151
# Build the tensor model-parallel groups.
136
- global _TP_DEVICE_GROUP, _TP_CPU_GROUP
152
+ global _TP_DEVICE_GROUP, _TP_CPU_GROUP, _TP_PYNCCL_COMMUNICATOR
153
assert _TP_DEVICE_GROUP is None, (
154
"tensor model parallel group is already initialized")
155
for i in range(num_tensor_model_parallel_groups):
140
- ranks = range(i * tensor_model_parallel_size,
141
- (i + 1) * tensor_model_parallel_size)
156
+ ranks = list(
157
+ range(i * tensor_model_parallel_size,
158
+ (i + 1) * tensor_model_parallel_size))
159
group = torch.distributed.new_group(ranks, backend=backend)
160
cpu_group = torch.distributed.new_group(ranks, backend="gloo")
161
if rank in ranks:
162
_TP_DEVICE_GROUP = group
163
_TP_CPU_GROUP = cpu_group
164
165
+ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
166
+ _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
167
+ group=_TP_CPU_GROUP,
168
+ device=_LOCAL_RANK,
169
+ )
170
+
171
# Build the pipeline model-parallel groups.
149
- global _PIPELINE_MODEL_PARALLEL_GROUP
150
- global _PIPELINE_GLOBAL_RANKS
151
- assert _PIPELINE_MODEL_PARALLEL_GROUP is None, (
172
+ global _PP_DEVICE_GROUP
173
+ global _PP_GLOBAL_RANKS
174
+ assert _PP_DEVICE_GROUP is None, (
175
"pipeline model parallel group is already initialized")
176
for i in range(num_pipeline_model_parallel_groups):
154
- ranks = range(i, world_size, num_pipeline_model_parallel_groups)
177
+ ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
178
group = torch.distributed.new_group(ranks, backend=backend)
179
if rank in ranks:
157
- _PIPELINE_MODEL_PARALLEL_GROUP = group
158
- _PIPELINE_GLOBAL_RANKS = ranks
180
+ _PP_DEVICE_GROUP = group
181
+ _PP_GLOBAL_RANKS = ranks
182
183
184
def ensure_model_parallel_initialized(
@@ -188,8 +211,7 @@ def ensure_model_parallel_initialized(
211
212
def model_parallel_is_initialized():
213
"""Check if tensor and pipeline parallel groups are initialized."""
191
- return (_TP_DEVICE_GROUP is not None
192
- and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
214
+ return (_TP_DEVICE_GROUP is not None and _PP_DEVICE_GROUP is not None)
215
216
217
def get_cpu_world_group():
@@ -214,9 +236,9 @@ def get_tensor_model_parallel_cpu_group():
236
237
def get_pipeline_model_parallel_group():
238
"""Get the pipeline model parallel group the caller rank belongs to."""
217
- assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, (
239
+ assert _PP_DEVICE_GROUP is not None, (
240
"pipeline model parallel group is not initialized")
219
- return _PIPELINE_MODEL_PARALLEL_GROUP
241
+ return _PP_DEVICE_GROUP
242
243
244
def get_tensor_model_parallel_world_size():
@@ -253,36 +275,36 @@ def get_tensor_model_parallel_src_rank():
275
def get_pipeline_model_parallel_first_rank():
276
"""Return the global rank of the first process in the pipeline for the
277
current tensor parallel group"""
256
- assert _PIPELINE_GLOBAL_RANKS is not None, (
278
+ assert _PP_GLOBAL_RANKS is not None, (
279
"Pipeline parallel group is not initialized")
258
- return _PIPELINE_GLOBAL_RANKS[0]
280
+ return _PP_GLOBAL_RANKS[0]
281
282
283
def get_pipeline_model_parallel_last_rank():
284
"""Return the global rank of the last process in the pipeline for the
285
current tensor parallel group"""
264
- assert _PIPELINE_GLOBAL_RANKS is not None, (
286
+ assert _PP_GLOBAL_RANKS is not None, (
287
"Pipeline parallel group is not initialized")
288
last_rank_local = get_pipeline_model_parallel_world_size() - 1
267
- return _PIPELINE_GLOBAL_RANKS[last_rank_local]
289
+ return _PP_GLOBAL_RANKS[last_rank_local]
290
291
292
def get_pipeline_model_parallel_next_rank():
293
"""Return the global rank that follows the caller in the pipeline"""
272
- assert _PIPELINE_GLOBAL_RANKS is not None, (
294
+ assert _PP_GLOBAL_RANKS is not None, (
295
"Pipeline parallel group is not initialized")
296
rank_in_pipeline = get_pipeline_model_parallel_rank()
297
world_size = get_pipeline_model_parallel_world_size()
276
- return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
298
+ return _PP_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
299
300
301
def get_pipeline_model_parallel_prev_rank():
302
"""Return the global rank that precedes the caller in the pipeline"""
281
- assert _PIPELINE_GLOBAL_RANKS is not None, (
303
+ assert _PP_GLOBAL_RANKS is not None, (
304
"Pipeline parallel group is not initialized")
305
rank_in_pipeline = get_pipeline_model_parallel_rank()
306
world_size = get_pipeline_model_parallel_world_size()
285
- return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
307
+ return _PP_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
308
309
310
def destroy_model_parallel():
@@ -295,45 +317,12 @@ def destroy_model_parallel():
317
if _TP_CPU_GROUP:
318
torch.distributed.destroy_process_group(_TP_CPU_GROUP)
319
_TP_CPU_GROUP = None
298
- global _PIPELINE_MODEL_PARALLEL_GROUP
299
- if _PIPELINE_MODEL_PARALLEL_GROUP:
300
- torch.distributed.destroy_process_group(_PIPELINE_MODEL_PARALLEL_GROUP)
301
- _PIPELINE_MODEL_PARALLEL_GROUP = None
302
- global _PIPELINE_GLOBAL_RANKS
303
- _PIPELINE_GLOBAL_RANKS = None
304
- from vllm.distributed.device_communicators import pynccl_utils
305
-
306
- # Destroy the pynccl states if any.
307
- pynccl_utils.destroy_process_group()
308
-
309
-
310
-# Whether to use pynccl for nccl all reduce.
311
-# We use pynccl for all reduce when using CUDA graph, because torch.distributed
312
-# is not well supported by CUDA graph.
313
-_ENABLE_PYNCCL_FOR_ALL_REDUCE = False
314
-
315
-
316
-@contextlib.contextmanager
317
-def with_pynccl_for_all_reduce():
318
- from vllm.distributed.device_communicators import pynccl_utils
319
- """use pynccl instead of torch.distributed for all reduce"""
320
- tp_size = get_tensor_model_parallel_world_size()
321
- if tp_size == 1:
322
- # No-op.
323
- # NOTE(woosuk): We don't initialize pynccl when tp_size is 1.
324
- yield
325
- else:
326
- global _ENABLE_PYNCCL_FOR_ALL_REDUCE
327
- old = _ENABLE_PYNCCL_FOR_ALL_REDUCE
328
- _ENABLE_PYNCCL_FOR_ALL_REDUCE = True
329
-
330
- stream = torch.cuda.current_stream()
331
- with pynccl_utils.set_pynccl_stream(stream):
332
- yield
333
- _ENABLE_PYNCCL_FOR_ALL_REDUCE = old
334
-
335
-
336
-def is_pynccl_enabled_for_all_reduce():
337
- """check if pynccl is enabled for all reduce"""
338
- global _ENABLE_PYNCCL_FOR_ALL_REDUCE
339
- return _ENABLE_PYNCCL_FOR_ALL_REDUCE
320
+ global _TP_PYNCCL_COMMUNICATOR
321
+ _TP_PYNCCL_COMMUNICATOR = None
322
+
323
+ global _PP_DEVICE_GROUP
324
+ if _PP_DEVICE_GROUP:
325
+ torch.distributed.destroy_process_group(_PP_DEVICE_GROUP)
326
+ _PP_DEVICE_GROUP = None
327
+ global _PP_GLOBAL_RANKS
328
+ _PP_GLOBAL_RANKS = None
vllm/worker/model_runner.py
+5
-20
@@ -1,4 +1,3 @@
1
-import contextlib
1
import time
2
from enum import IntEnum
3
from typing import Dict, List, NamedTuple, Optional, Set, Tuple
@@ -12,9 +11,9 @@ from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
ModelConfig, ParallelConfig, SchedulerConfig,
13
VisionLanguageConfig)
15
-from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
16
-from vllm.distributed.device_communicators import (custom_all_reduce,
17
- pynccl_utils)
14
+from vllm.distributed import broadcast_tensor_dict
15
+from vllm.distributed.communication_op import graph_capture_mode
16
+from vllm.distributed.device_communicators import custom_all_reduce
17
from vllm.logger import init_logger
18
from vllm.lora.layers import LoRAMapping
19
from vllm.lora.request import LoRARequest
@@ -917,10 +916,6 @@ class ModelRunner:
916
Since it is used for decoding-only, it assumes there's only 1 token
917
per sequence in the batch.
918
"""
920
- # NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
921
- # deleted before the CUDA graphs.
922
- self.pynccl_backend = pynccl_utils.get_nccl_backend()
923
-
919
assert not self.model_config.enforce_eager
920
logger.info("Capturing the model for CUDA graphs. This may lead to "
921
"unexpected consequences if the model is not static. To "
@@ -1046,7 +1041,7 @@ class CUDAGraphRunner:
1041
# Run the model once without capturing the graph.
1042
# This is to make sure that the captured graph does not include the
1043
# kernel launches for initial benchmarking (e.g., Triton autotune).
1049
- with _maybe_pynccl():
1044
+ with graph_capture_mode():
1045
self.model(
1046
input_ids,
1047
positions,
@@ -1061,7 +1056,7 @@ class CUDAGraphRunner:
1056
# https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
1057
self._graph = torch.cuda.CUDAGraph()
1058
with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
1064
- with _maybe_pynccl():
1059
+ with graph_capture_mode():
1060
hidden_states = self.model(
1061
input_ids,
1062
positions,
@@ -1113,16 +1108,6 @@ class CUDAGraphRunner:
1108
return self.forward(*args, **kwargs)
1109
1110
1116
-@contextlib.contextmanager
1117
-def _maybe_pynccl():
1118
- if pynccl_utils.is_initialized(
1119
- ) and not custom_all_reduce.is_initialized():
1120
- with with_pynccl_for_all_reduce():
1121
- yield
1122
- else:
1123
- yield
1124
-
1125
-
1111
def _get_graph_batch_size(batch_size: int) -> int:
1112
"""Returns the padded batch size given actual batch size.
1113
vllm/worker/worker.py
-21
@@ -11,9 +11,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
VisionLanguageConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
14
- get_tensor_model_parallel_cpu_group,
14
init_distributed_environment)
16
-from vllm.distributed.device_communicators import pynccl_utils
15
from vllm.distributed.device_communicators.custom_all_reduce import (
16
init_custom_ar)
17
from vllm.lora.request import LoRARequest
@@ -306,29 +304,10 @@ def init_worker_distributed_environment(
304
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
305
parallel_config.pipeline_parallel_size)
306
309
- if pynccl_utils.is_initialized():
310
- pynccl_world_size = pynccl_utils.get_world_size()
311
- if pynccl_world_size != parallel_config.world_size:
312
- raise RuntimeError(
313
- "pynccl is already initialized but the pynccl world "
314
- "size does not match parallel_config.world_size "
315
- f"({pynccl_world_size} vs. {parallel_config.world_size}).")
316
- elif parallel_config.world_size > 1:
317
- # NOTE(woosuk): We don't initialize pynccl process group when world size
318
- # is 1.
319
- # NOTE(kaichao): By default, pynccl is initialized for tp group.
320
- pynccl_utils.init_process_group(
321
- group=get_tensor_model_parallel_cpu_group())
322
-
307
# Initialize a custom fast all-reduce implementation.
308
if not parallel_config.disable_custom_all_reduce:
309
init_custom_ar()
310
327
- # A small all_reduce for warmup.
328
- torch.distributed.all_reduce(torch.zeros(1).cuda())
329
- if pynccl_utils.is_initialized():
330
- pynccl_utils.all_reduce(torch.zeros(1).cuda())
331
-
311
312
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
313
# Check if the GPU supports the dtype.