@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 978b39744b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Misc] Add pynccl wrappers for all_gather and reduce_scatter (#9432)

Tyler Michael Smith committed Nov 22, 2024 at 22:14 UTC 978b39744b22e90d49a0f5367c3d933ed26d66c8
3 files changed +155
tests/distributed/test_pynccl.py
+69
@@ -150,6 +150,75 @@ def worker_fn_with_cudagraph():
150 assert a.mean().cpu().item() == pynccl_comm.world_size**1
151
152
153 +@worker_fn_wrapper
154 +def all_gather_worker_fn():
155 + pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
156 + device=get_world_group().device)
157 +
158 + rank = pynccl_comm.rank
159 + world_size = pynccl_comm.world_size
160 + device = f'cuda:{pynccl_comm.rank}'
161 +
162 + num_elems = 1000
163 + tensor = torch.arange(num_elems, dtype=torch.float32,
164 + device=device) + rank * num_elems
165 + result = torch.zeros(num_elems * world_size,
166 + dtype=torch.float32,
167 + device=device)
168 +
169 + expected = torch.cat([
170 + torch.arange(num_elems, dtype=torch.float32) + r * num_elems
171 + for r in range(world_size)
172 + ]).to(device)
173 +
174 + with pynccl_comm.change_state(enable=True):
175 + pynccl_comm.all_gather(result, tensor)
176 + torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
177 +
178 +
179 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
180 + reason="Need at least 2 GPUs to run the test.")
181 +def test_pynccl_all_gather():
182 + distributed_run(all_gather_worker_fn, 2)
183 +
184 +
185 +@worker_fn_wrapper
186 +def reduce_scatter_worker_fn():
187 + pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
188 + device=get_world_group().device)
189 +
190 + rank = pynccl_comm.rank
191 + world_size = pynccl_comm.world_size
192 + device = f'cuda:{pynccl_comm.rank}'
193 +
194 + num_elems = 1000
195 + tensor = torch.arange(num_elems, dtype=torch.float32,
196 + device=device) + rank * num_elems
197 + assert (num_elems % world_size == 0)
198 + result = torch.zeros(num_elems // world_size,
199 + dtype=torch.float32,
200 + device=device)
201 +
202 + # Calculate expected result for this rank's chunk
203 + scattered_size = num_elems // world_size
204 + all_tensors = [
205 + torch.arange(num_elems, dtype=torch.float32) + r * num_elems
206 + for r in range(world_size)
207 + ]
208 + expected = sum(tensor[rank * scattered_size:(rank + 1) * scattered_size]
209 + for tensor in all_tensors).to(device)
210 +
211 + with pynccl_comm.change_state(enable=True):
212 + pynccl_comm.reduce_scatter(result, tensor)
213 + torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
214 +
215 +
216 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
217 + reason="Need at least 2 GPUs to run the test.")
218 +def test_pynccl_reduce_scatter():
219 + distributed_run(reduce_scatter_worker_fn, 2)
220 +
221 +
222 @pytest.mark.skipif(torch.cuda.device_count() < 2,
223 reason="Need at least 2 GPUs to run the test.")
224 def test_pynccl_with_cudagraph():
vllm/distributed/device_communicators/pynccl.py
+42
@@ -131,6 +131,48 @@ class PyNcclCommunicator:
131 ncclRedOpTypeEnum.from_torch(op), self.comm,
132 cudaStream_t(stream.cuda_stream))
133
134 + def all_gather(self,
135 + output_tensor: torch.Tensor,
136 + input_tensor: torch.Tensor,
137 + stream=None):
138 + if self.disabled:
139 + return
140 + # nccl communicator created on a specific device
141 + # will only work on tensors on the same device
142 + # otherwise it will cause "illegal memory access"
143 + assert input_tensor.device == self.device, (
144 + f"this nccl communicator is created to work on {self.device}, "
145 + f"but the input tensor is on {input_tensor.device}")
146 + if stream is None:
147 + stream = self.stream
148 + self.nccl.ncclAllGather(
149 + buffer_type(input_tensor.data_ptr()),
150 + buffer_type(output_tensor.data_ptr()), input_tensor.numel(),
151 + ncclDataTypeEnum.from_torch(input_tensor.dtype), self.comm,
152 + cudaStream_t(stream.cuda_stream))
153 +
154 + def reduce_scatter(self,
155 + output_tensor: torch.Tensor,
156 + input_tensor: torch.Tensor,
157 + op: ReduceOp = ReduceOp.SUM,
158 + stream=None):
159 + if self.disabled:
160 + return
161 + # nccl communicator created on a specific device
162 + # will only work on tensors on the same device
163 + # otherwise it will cause "illegal memory access"
164 + assert input_tensor.device == self.device, (
165 + f"this nccl communicator is created to work on {self.device}, "
166 + f"but the input tensor is on {input_tensor.device}")
167 + if stream is None:
168 + stream = self.stream
169 + self.nccl.ncclReduceScatter(
170 + buffer_type(input_tensor.data_ptr()),
171 + buffer_type(output_tensor.data_ptr()), output_tensor.numel(),
172 + ncclDataTypeEnum.from_torch(input_tensor.dtype),
173 + ncclRedOpTypeEnum.from_torch(op), self.comm,
174 + cudaStream_t(stream.cuda_stream))
175 +
176 def send(self, tensor: torch.Tensor, dst: int, stream=None):
177 if self.disabled:
178 return
vllm/distributed/device_communicators/pynccl_wrapper.py
+44
@@ -151,6 +151,28 @@ class NCCLLibrary:
151 ncclRedOp_t, ncclComm_t, cudaStream_t
152 ]),
153
154 + # ncclResult_t ncclAllGather(
155 + # const void* sendbuff, void* recvbuff, size_t count,
156 + # ncclDataType_t datatype, ncclComm_t comm,
157 + # cudaStream_t stream);
158 + # note that cudaStream_t is a pointer type, so the last argument
159 + # is a pointer
160 + Function("ncclAllGather", ncclResult_t, [
161 + buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
162 + ncclComm_t, cudaStream_t
163 + ]),
164 +
165 + # ncclResult_t ncclReduceScatter(
166 + # const void* sendbuff, void* recvbuff, size_t count,
167 + # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
168 + # cudaStream_t stream);
169 + # note that cudaStream_t is a pointer type, so the last argument
170 + # is a pointer
171 + Function("ncclReduceScatter", ncclResult_t, [
172 + buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
173 + ncclRedOp_t, ncclComm_t, cudaStream_t
174 + ]),
175 +
176 # ncclResult_t ncclSend(
177 # const void* sendbuff, size_t count, ncclDataType_t datatype,
178 # int dest, ncclComm_t comm, cudaStream_t stream);
@@ -258,6 +280,28 @@ class NCCLLibrary:
280 datatype, op, comm,
281 stream))
282
283 + def ncclReduceScatter(self, sendbuff: buffer_type, recvbuff: buffer_type,
284 + count: int, datatype: int, op: int, comm: ncclComm_t,
285 + stream: cudaStream_t) -> None:
286 + # `datatype` actually should be `ncclDataType_t`
287 + # and `op` should be `ncclRedOp_t`
288 + # both are aliases of `ctypes.c_int`
289 + # when we pass int to a function, it will be converted to `ctypes.c_int`
290 + # by ctypes automatically
291 + self.NCCL_CHECK(self._funcs["ncclReduceScatter"](sendbuff, recvbuff,
292 + count, datatype, op,
293 + comm, stream))
294 +
295 + def ncclAllGather(self, sendbuff: buffer_type, recvbuff: buffer_type,
296 + count: int, datatype: int, comm: ncclComm_t,
297 + stream: cudaStream_t) -> None:
298 + # `datatype` actually should be `ncclDataType_t`
299 + # which is an aliases of `ctypes.c_int`
300 + # when we pass int to a function, it will be converted to `ctypes.c_int`
301 + # by ctypes automatically
302 + self.NCCL_CHECK(self._funcs["ncclAllGather"](sendbuff, recvbuff, count,
303 + datatype, comm, stream))
304 +
305 def ncclSend(self, sendbuff: buffer_type, count: int, datatype: int,
306 dest: int, comm: ncclComm_t, stream: cudaStream_t) -> None:
307 self.NCCL_CHECK(self._funcs["ncclSend"](sendbuff, count, datatype,