A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core][1/N] Support send/recv in PyNCCL Groups (#4988)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
Murali Andoorveedu committed
May 23, 2024 at 09:54 UTC
5eda2ea02a01b2457f4d6ac2a217f2fa8a2e5d5f
5 files changed
+170
-17
tests/distributed/test_pynccl.py
+69
-6
@@ -3,6 +3,7 @@ import os
3
4
import pytest
5
import torch
6
+import torch.distributed
7
8
from vllm.distributed.communication_op import ( # noqa
9
graph_capture, tensor_model_parallel_all_reduce)
@@ -68,7 +69,7 @@ def test_pynccl():
69
70
71
@worker_fn_wrapper
71
-def multiple_tp_worker_fn():
72
+def multiple_allreduce_worker_fn():
73
device = torch.device(f"cuda:{torch.distributed.get_rank()}")
74
groups = [
75
torch.distributed.new_group(ranks=[0, 1], backend="gloo"),
@@ -92,14 +93,14 @@ def multiple_tp_worker_fn():
93
94
@pytest.mark.skipif(torch.cuda.device_count() < 4,
95
reason="Need at least 4 GPUs to run the test.")
95
-def test_pynccl_multiple_tp():
96
+def test_pynccl_multiple_allreduce():
97
# this tests pynccl for multiple tp groups, in a standalone way
98
# i.e. call `pynccl_comm.all_reduce` directly
98
- distributed_run(multiple_tp_worker_fn, 4)
99
+ distributed_run(multiple_allreduce_worker_fn, 4)
100
101
102
@worker_fn_wrapper
102
-def multiple_tp_with_vllm_worker_fn():
103
+def multiple_allreduce_with_vllm_worker_fn():
104
device = torch.device(f"cuda:{torch.distributed.get_rank()}")
105
ensure_model_parallel_initialized(2, 2)
106
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
@@ -118,10 +119,10 @@ def multiple_tp_with_vllm_worker_fn():
119
120
@pytest.mark.skipif(torch.cuda.device_count() < 4,
121
reason="Need at least 4 GPUs to run the test.")
121
-def test_pynccl_multiple_tp_with_vllm():
122
+def test_pynccl_multiple_allreduce_with_vllm():
123
# this tests pynccl for multiple tp groups, together with vllm
124
# i.e. call `tensor_model_parallel_all_reduce`
124
- distributed_run(multiple_tp_with_vllm_worker_fn, 4)
125
+ distributed_run(multiple_allreduce_with_vllm_worker_fn, 4)
126
127
128
@worker_fn_wrapper
@@ -151,6 +152,68 @@ def test_pynccl_with_cudagraph():
152
distributed_run(worker_fn_with_cudagraph, 2)
153
154
155
+@worker_fn_wrapper
156
+def send_recv_worker_fn():
157
+ pynccl_comm = PyNcclCommunicator()
158
+ if pynccl_comm.rank == 0:
159
+ tensor = torch.ones(16, 1024, 1024,
160
+ dtype=torch.float32).cuda(pynccl_comm.rank)
161
+ else:
162
+ tensor = torch.empty(16, 1024, 1024,
163
+ dtype=torch.float32).cuda(pynccl_comm.rank)
164
+ with pynccl_comm.change_state(enable=True):
165
+ if pynccl_comm.rank == 0:
166
+ pynccl_comm.send(tensor)
167
+ else:
168
+ pynccl_comm.recv(tensor)
169
+ result = tensor.mean().cpu().item()
170
+ assert result == 1
171
+
172
+
173
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
174
+ reason="Need at least 2 GPUs to run the test.")
175
+def test_pynccl_send_recv():
176
+ distributed_run(send_recv_worker_fn, 2)
177
+
178
+
179
+@worker_fn_wrapper
180
+def multiple_send_recv_worker_fn():
181
+ device = torch.device(f"cuda:{torch.distributed.get_rank()}")
182
+ groups = [
183
+ torch.distributed.new_group(ranks=[0, 2], backend="gloo"),
184
+ torch.distributed.new_group(ranks=[1, 3], backend="gloo")
185
+ ]
186
+ group = groups[0] if torch.distributed.get_rank() in [0, 2] else groups[1]
187
+ pynccl_comm = PyNcclCommunicator(group=group, device=device)
188
+ if torch.distributed.get_rank() == 0:
189
+ tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
190
+ elif torch.distributed.get_rank() == 1:
191
+ tensor = 2 * torch.ones(
192
+ 16, 1024, 1024, dtype=torch.float32, device=device)
193
+ else:
194
+ tensor = torch.empty(16,
195
+ 1024,
196
+ 1024,
197
+ dtype=torch.float32,
198
+ device=device)
199
+ with pynccl_comm.change_state(enable=True):
200
+ if torch.distributed.get_rank() in [0, 1]:
201
+ pynccl_comm.send(tensor)
202
+ else:
203
+ pynccl_comm.recv(tensor)
204
+ result = tensor.mean().cpu().item()
205
+ if torch.distributed.get_rank() in [0, 2]:
206
+ assert result == 1
207
+ else:
208
+ assert result == 2
209
+
210
+
211
+@pytest.mark.skipif(torch.cuda.device_count() < 4,
212
+ reason="Need at least 4 GPUs to run the test.")
213
+def test_pynccl_multiple_send_recv():
214
+ distributed_run(multiple_send_recv_worker_fn, 4)
215
+
216
+
217
def test_ncclGetUniqueId():
218
lib = NCCLLibrary()
219
unique_id = lib.ncclGetUniqueId()
vllm/distributed/communication_op.py
+12
-6
@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
6
import torch
7
from torch.distributed import ProcessGroup
8
9
-from .parallel_state import (get_cpu_world_group,
9
+from .parallel_state import (get_cpu_world_group, get_pp_pynccl_communicator,
10
get_tensor_model_parallel_group,
11
get_tensor_model_parallel_rank,
12
get_tensor_model_parallel_world_size,
@@ -54,13 +54,19 @@ def graph_capture():
54
# graph, we use either custom all-reduce kernel or PyTorch NCCL.
55
# We always prioritize using custom all-reduce kernel but fall back
56
# to PyTorch or pynccl if it is disabled or not supported.
57
- pynccl_comm = get_tp_pynccl_communicator()
58
- if pynccl_comm is None:
59
- maybe_pynccl_context = nullcontext()
57
+ tp_pynccl_comm = get_tp_pynccl_communicator()
58
+ pp_pynccl_comm = get_pp_pynccl_communicator()
59
+ if not tp_pynccl_comm:
60
+ maybe_tp_pynccl_context = nullcontext()
61
else:
61
- maybe_pynccl_context = pynccl_comm.change_state(
62
+ maybe_tp_pynccl_context = tp_pynccl_comm.change_state(
63
enable=True, stream=torch.cuda.current_stream())
63
- with maybe_pynccl_context:
64
+ if not pp_pynccl_comm:
65
+ maybe_pp_pynccl_context = nullcontext()
66
+ else:
67
+ maybe_pp_pynccl_context = pp_pynccl_comm.change_state(
68
+ enable=True, stream=torch.cuda.current_stream())
69
+ with maybe_tp_pynccl_context, maybe_pp_pynccl_context:
70
yield graph_capture_context
71
72
vllm/distributed/device_communicators/pynccl.py
+34
@@ -126,6 +126,40 @@ class PyNcclCommunicator:
126
ncclRedOpTypeEnum.from_torch(op), self.comm,
127
cudaStream_t(stream.cuda_stream))
128
129
+ def send(self,
130
+ tensor: torch.Tensor,
131
+ dst: Optional[int] = None,
132
+ stream=None):
133
+ if self.disabled:
134
+ return
135
+ assert tensor.device == self.device, (
136
+ f"this nccl communicator is created to work on {self.device}, "
137
+ f"but the input tensor is on {tensor.device}")
138
+ if stream is None:
139
+ stream = self.stream
140
+ if dst is None:
141
+ dst = (self.rank + 1) % self.world_size
142
+ self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
143
+ ncclDataTypeEnum.from_torch(tensor.dtype), dst,
144
+ self.comm, cudaStream_t(stream.cuda_stream))
145
+
146
+ def recv(self,
147
+ tensor: torch.Tensor,
148
+ src: Optional[int] = None,
149
+ stream=None):
150
+ if self.disabled:
151
+ return
152
+ assert tensor.device == self.device, (
153
+ f"this nccl communicator is created to work on {self.device}, "
154
+ f"but the input tensor is on {tensor.device}")
155
+ if stream is None:
156
+ stream = self.stream
157
+ if src is None:
158
+ src = (self.rank - 1) % self.world_size
159
+ self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
160
+ ncclDataTypeEnum.from_torch(tensor.dtype), src,
161
+ self.comm, cudaStream_t(stream.cuda_stream))
162
+
163
@contextmanager
164
def change_state(self,
165
enable: Optional[bool] = None,
vllm/distributed/device_communicators/pynccl_wrapper.py
+26
@@ -151,6 +151,22 @@ class NCCLLibrary:
151
ncclRedOp_t, ncclComm_t, cudaStream_t
152
]),
153
154
+ # ncclResult_t ncclSend(
155
+ # const void* sendbuff, size_t count, ncclDataType_t datatype,
156
+ # int dest, ncclComm_t comm, cudaStream_t stream);
157
+ Function("ncclSend", ncclResult_t, [
158
+ buffer_type, ctypes.c_size_t, ncclDataType_t, ctypes.c_int,
159
+ ncclComm_t, cudaStream_t
160
+ ]),
161
+
162
+ # ncclResult_t ncclRecv(
163
+ # void* recvbuff, size_t count, ncclDataType_t datatype,
164
+ # int src, ncclComm_t comm, cudaStream_t stream);
165
+ Function("ncclRecv", ncclResult_t, [
166
+ buffer_type, ctypes.c_size_t, ncclDataType_t, ctypes.c_int,
167
+ ncclComm_t, cudaStream_t
168
+ ]),
169
+
170
# be cautious! this is a collective call, it will block until all
171
# processes in the communicator have called this function.
172
# because Python object destruction can happen in random order,
@@ -248,6 +264,16 @@ class NCCLLibrary:
264
datatype, op, comm,
265
stream))
266
267
+ def ncclSend(self, sendbuff: buffer_type, count: int, datatype: int,
268
+ dest: int, comm: ncclComm_t, stream: cudaStream_t) -> None:
269
+ self.NCCL_CHECK(self._funcs["ncclSend"](sendbuff, count, datatype,
270
+ dest, comm, stream))
271
+
272
+ def ncclRecv(self, recvbuff: buffer_type, count: int, datatype: int,
273
+ src: int, comm: ncclComm_t, stream: cudaStream_t) -> None:
274
+ self.NCCL_CHECK(self._funcs["ncclRecv"](recvbuff, count, datatype, src,
275
+ comm, stream))
276
+
277
def ncclCommDestroy(self, comm: ncclComm_t) -> None:
278
self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
279
vllm/distributed/parallel_state.py
+29
-5
@@ -22,6 +22,8 @@ _TP_PYNCCL_COMMUNICATOR = None
22
_TP_CA_COMMUNICATOR = None
23
# Pipeline model parallel group that the current rank belongs to.
24
_PP_DEVICE_GROUP: Optional[ProcessGroup] = None
25
+_PP_CPU_GROUP: Optional[ProcessGroup] = None
26
+_PP_PYNCCL_COMMUNICATOR = None
27
28
# when people blindly call `torch.distributed.all_reduce` etc,
29
# it will use this group. It is initialized with the `backend`
@@ -55,6 +57,11 @@ def set_custom_all_reduce(enable: bool):
57
_ENABLE_CUSTOM_ALL_REDUCE = enable
58
59
60
+def get_pp_pynccl_communicator():
61
+ global _PP_PYNCCL_COMMUNICATOR
62
+ return _PP_PYNCCL_COMMUNICATOR
63
+
64
+
65
def get_tp_pynccl_communicator():
66
global _TP_PYNCCL_COMMUNICATOR
67
return _TP_PYNCCL_COMMUNICATOR
@@ -180,10 +187,11 @@ def initialize_model_parallel(
187
_TP_CPU_GROUP = cpu_group
188
189
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
183
- _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
184
- group=_TP_CPU_GROUP,
185
- device=_LOCAL_RANK,
186
- )
190
+ if tensor_model_parallel_size > 1:
191
+ _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
192
+ group=_TP_CPU_GROUP,
193
+ device=_LOCAL_RANK,
194
+ )
195
196
# Initialize a custom fast all-reduce implementation.
197
if _ENABLE_CUSTOM_ALL_REDUCE:
@@ -195,17 +203,26 @@ def initialize_model_parallel(
203
)
204
205
# Build the pipeline model-parallel groups.
198
- global _PP_DEVICE_GROUP
206
+ global _PP_DEVICE_GROUP, _PP_CPU_GROUP
207
+ global _PP_PYNCCL_COMMUNICATOR
208
global _PP_GLOBAL_RANKS
209
assert _PP_DEVICE_GROUP is None, (
210
"pipeline model parallel group is already initialized")
211
for i in range(num_pipeline_model_parallel_groups):
212
ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
213
group = torch.distributed.new_group(ranks, backend=backend)
214
+ cpu_group = torch.distributed.new_group(ranks, backend="gloo")
215
if rank in ranks:
216
_PP_DEVICE_GROUP = group
217
+ _PP_CPU_GROUP = cpu_group
218
_PP_GLOBAL_RANKS = ranks
219
220
+ if pipeline_model_parallel_size > 1:
221
+ _PP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
222
+ group=_PP_CPU_GROUP,
223
+ device=_LOCAL_RANK,
224
+ )
225
+
226
227
def ensure_model_parallel_initialized(
228
tensor_model_parallel_size: int,
@@ -267,6 +284,13 @@ def get_pipeline_model_parallel_group():
284
return _PP_DEVICE_GROUP
285
286
287
+def get_pipeline_model_parallel_cpu_group():
288
+ """Get the pipeline model parallel cpu group the caller rank belongs to."""
289
+ assert _PP_CPU_GROUP is not None, (
290
+ "pipeline model parallel cpu group is not initialized")
291
+ return _PP_CPU_GROUP
292
+
293
+
294
def get_tensor_model_parallel_world_size():
295
"""Return world size for the tensor model parallel group."""
296
return torch.distributed.get_world_size(