A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Misc] Add pynccl wrappers for all_gather and reduce_scatter (#9432)
Tyler Michael Smith committed
Nov 22, 2024 at 22:14 UTC
978b39744b22e90d49a0f5367c3d933ed26d66c8
3 files changed
+155
tests/distributed/test_pynccl.py
+69
@@ -150,6 +150,75 @@ def worker_fn_with_cudagraph():
150
assert a.mean().cpu().item() == pynccl_comm.world_size**1
151
152
153
+@worker_fn_wrapper
154
+def all_gather_worker_fn():
155
+ pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
156
+ device=get_world_group().device)
157
+
158
+ rank = pynccl_comm.rank
159
+ world_size = pynccl_comm.world_size
160
+ device = f'cuda:{pynccl_comm.rank}'
161
+
162
+ num_elems = 1000
163
+ tensor = torch.arange(num_elems, dtype=torch.float32,
164
+ device=device) + rank * num_elems
165
+ result = torch.zeros(num_elems * world_size,
166
+ dtype=torch.float32,
167
+ device=device)
168
+
169
+ expected = torch.cat([
170
+ torch.arange(num_elems, dtype=torch.float32) + r * num_elems
171
+ for r in range(world_size)
172
+ ]).to(device)
173
+
174
+ with pynccl_comm.change_state(enable=True):
175
+ pynccl_comm.all_gather(result, tensor)
176
+ torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
177
+
178
+
179
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
180
+ reason="Need at least 2 GPUs to run the test.")
181
+def test_pynccl_all_gather():
182
+ distributed_run(all_gather_worker_fn, 2)
183
+
184
+
185
+@worker_fn_wrapper
186
+def reduce_scatter_worker_fn():
187
+ pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
188
+ device=get_world_group().device)
189
+
190
+ rank = pynccl_comm.rank
191
+ world_size = pynccl_comm.world_size
192
+ device = f'cuda:{pynccl_comm.rank}'
193
+
194
+ num_elems = 1000
195
+ tensor = torch.arange(num_elems, dtype=torch.float32,
196
+ device=device) + rank * num_elems
197
+ assert (num_elems % world_size == 0)
198
+ result = torch.zeros(num_elems // world_size,
199
+ dtype=torch.float32,
200
+ device=device)
201
+
202
+ # Calculate expected result for this rank's chunk
203
+ scattered_size = num_elems // world_size
204
+ all_tensors = [
205
+ torch.arange(num_elems, dtype=torch.float32) + r * num_elems
206
+ for r in range(world_size)
207
+ ]
208
+ expected = sum(tensor[rank * scattered_size:(rank + 1) * scattered_size]
209
+ for tensor in all_tensors).to(device)
210
+
211
+ with pynccl_comm.change_state(enable=True):
212
+ pynccl_comm.reduce_scatter(result, tensor)
213
+ torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
214
+
215
+
216
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
217
+ reason="Need at least 2 GPUs to run the test.")
218
+def test_pynccl_reduce_scatter():
219
+ distributed_run(reduce_scatter_worker_fn, 2)
220
+
221
+
222
@pytest.mark.skipif(torch.cuda.device_count() < 2,
223
reason="Need at least 2 GPUs to run the test.")
224
def test_pynccl_with_cudagraph():
vllm/distributed/device_communicators/pynccl.py
+42
@@ -131,6 +131,48 @@ class PyNcclCommunicator:
131
ncclRedOpTypeEnum.from_torch(op), self.comm,
132
cudaStream_t(stream.cuda_stream))
133
134
+ def all_gather(self,
135
+ output_tensor: torch.Tensor,
136
+ input_tensor: torch.Tensor,
137
+ stream=None):
138
+ if self.disabled:
139
+ return
140
+ # nccl communicator created on a specific device
141
+ # will only work on tensors on the same device
142
+ # otherwise it will cause "illegal memory access"
143
+ assert input_tensor.device == self.device, (
144
+ f"this nccl communicator is created to work on {self.device}, "
145
+ f"but the input tensor is on {input_tensor.device}")
146
+ if stream is None:
147
+ stream = self.stream
148
+ self.nccl.ncclAllGather(
149
+ buffer_type(input_tensor.data_ptr()),
150
+ buffer_type(output_tensor.data_ptr()), input_tensor.numel(),
151
+ ncclDataTypeEnum.from_torch(input_tensor.dtype), self.comm,
152
+ cudaStream_t(stream.cuda_stream))
153
+
154
+ def reduce_scatter(self,
155
+ output_tensor: torch.Tensor,
156
+ input_tensor: torch.Tensor,
157
+ op: ReduceOp = ReduceOp.SUM,
158
+ stream=None):
159
+ if self.disabled:
160
+ return
161
+ # nccl communicator created on a specific device
162
+ # will only work on tensors on the same device
163
+ # otherwise it will cause "illegal memory access"
164
+ assert input_tensor.device == self.device, (
165
+ f"this nccl communicator is created to work on {self.device}, "
166
+ f"but the input tensor is on {input_tensor.device}")
167
+ if stream is None:
168
+ stream = self.stream
169
+ self.nccl.ncclReduceScatter(
170
+ buffer_type(input_tensor.data_ptr()),
171
+ buffer_type(output_tensor.data_ptr()), output_tensor.numel(),
172
+ ncclDataTypeEnum.from_torch(input_tensor.dtype),
173
+ ncclRedOpTypeEnum.from_torch(op), self.comm,
174
+ cudaStream_t(stream.cuda_stream))
175
+
176
def send(self, tensor: torch.Tensor, dst: int, stream=None):
177
if self.disabled:
178
return
vllm/distributed/device_communicators/pynccl_wrapper.py
+44
@@ -151,6 +151,28 @@ class NCCLLibrary:
151
ncclRedOp_t, ncclComm_t, cudaStream_t
152
]),
153
154
+ # ncclResult_t ncclAllGather(
155
+ # const void* sendbuff, void* recvbuff, size_t count,
156
+ # ncclDataType_t datatype, ncclComm_t comm,
157
+ # cudaStream_t stream);
158
+ # note that cudaStream_t is a pointer type, so the last argument
159
+ # is a pointer
160
+ Function("ncclAllGather", ncclResult_t, [
161
+ buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
162
+ ncclComm_t, cudaStream_t
163
+ ]),
164
+
165
+ # ncclResult_t ncclReduceScatter(
166
+ # const void* sendbuff, void* recvbuff, size_t count,
167
+ # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
168
+ # cudaStream_t stream);
169
+ # note that cudaStream_t is a pointer type, so the last argument
170
+ # is a pointer
171
+ Function("ncclReduceScatter", ncclResult_t, [
172
+ buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
173
+ ncclRedOp_t, ncclComm_t, cudaStream_t
174
+ ]),
175
+
176
# ncclResult_t ncclSend(
177
# const void* sendbuff, size_t count, ncclDataType_t datatype,
178
# int dest, ncclComm_t comm, cudaStream_t stream);
@@ -258,6 +280,28 @@ class NCCLLibrary:
280
datatype, op, comm,
281
stream))
282
283
+ def ncclReduceScatter(self, sendbuff: buffer_type, recvbuff: buffer_type,
284
+ count: int, datatype: int, op: int, comm: ncclComm_t,
285
+ stream: cudaStream_t) -> None:
286
+ # `datatype` actually should be `ncclDataType_t`
287
+ # and `op` should be `ncclRedOp_t`
288
+ # both are aliases of `ctypes.c_int`
289
+ # when we pass int to a function, it will be converted to `ctypes.c_int`
290
+ # by ctypes automatically
291
+ self.NCCL_CHECK(self._funcs["ncclReduceScatter"](sendbuff, recvbuff,
292
+ count, datatype, op,
293
+ comm, stream))
294
+
295
+ def ncclAllGather(self, sendbuff: buffer_type, recvbuff: buffer_type,
296
+ count: int, datatype: int, comm: ncclComm_t,
297
+ stream: cudaStream_t) -> None:
298
+ # `datatype` actually should be `ncclDataType_t`
299
+ # which is an aliases of `ctypes.c_int`
300
+ # when we pass int to a function, it will be converted to `ctypes.c_int`
301
+ # by ctypes automatically
302
+ self.NCCL_CHECK(self._funcs["ncclAllGather"](sendbuff, recvbuff, count,
303
+ datatype, comm, stream))
304
+
305
def ncclSend(self, sendbuff: buffer_type, count: int, datatype: int,
306
dest: int, comm: ncclComm_t, stream: cudaStream_t) -> None:
307
self.NCCL_CHECK(self._funcs["ncclSend"](sendbuff, count, datatype,