@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5eda2ea02a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core][1/N] Support send/recv in PyNCCL Groups (#4988)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>

Murali Andoorveedu committed May 23, 2024 at 09:54 UTC 5eda2ea02a01b2457f4d6ac2a217f2fa8a2e5d5f
5 files changed +170 -17
tests/distributed/test_pynccl.py
+69 -6
@@ -3,6 +3,7 @@ import os
3
4 import pytest
5 import torch
6 +import torch.distributed
7
8 from vllm.distributed.communication_op import ( # noqa
9 graph_capture, tensor_model_parallel_all_reduce)
@@ -68,7 +69,7 @@ def test_pynccl():
69
70
71 @worker_fn_wrapper
71 -def multiple_tp_worker_fn():
72 +def multiple_allreduce_worker_fn():
73 device = torch.device(f"cuda:{torch.distributed.get_rank()}")
74 groups = [
75 torch.distributed.new_group(ranks=[0, 1], backend="gloo"),
@@ -92,14 +93,14 @@ def multiple_tp_worker_fn():
93
94 @pytest.mark.skipif(torch.cuda.device_count() < 4,
95 reason="Need at least 4 GPUs to run the test.")
95 -def test_pynccl_multiple_tp():
96 +def test_pynccl_multiple_allreduce():
97 # this tests pynccl for multiple tp groups, in a standalone way
98 # i.e. call `pynccl_comm.all_reduce` directly
98 - distributed_run(multiple_tp_worker_fn, 4)
99 + distributed_run(multiple_allreduce_worker_fn, 4)
100
101
102 @worker_fn_wrapper
102 -def multiple_tp_with_vllm_worker_fn():
103 +def multiple_allreduce_with_vllm_worker_fn():
104 device = torch.device(f"cuda:{torch.distributed.get_rank()}")
105 ensure_model_parallel_initialized(2, 2)
106 tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
@@ -118,10 +119,10 @@ def multiple_tp_with_vllm_worker_fn():
119
120 @pytest.mark.skipif(torch.cuda.device_count() < 4,
121 reason="Need at least 4 GPUs to run the test.")
121 -def test_pynccl_multiple_tp_with_vllm():
122 +def test_pynccl_multiple_allreduce_with_vllm():
123 # this tests pynccl for multiple tp groups, together with vllm
124 # i.e. call `tensor_model_parallel_all_reduce`
124 - distributed_run(multiple_tp_with_vllm_worker_fn, 4)
125 + distributed_run(multiple_allreduce_with_vllm_worker_fn, 4)
126
127
128 @worker_fn_wrapper
@@ -151,6 +152,68 @@ def test_pynccl_with_cudagraph():
152 distributed_run(worker_fn_with_cudagraph, 2)
153
154
155 +@worker_fn_wrapper
156 +def send_recv_worker_fn():
157 + pynccl_comm = PyNcclCommunicator()
158 + if pynccl_comm.rank == 0:
159 + tensor = torch.ones(16, 1024, 1024,
160 + dtype=torch.float32).cuda(pynccl_comm.rank)
161 + else:
162 + tensor = torch.empty(16, 1024, 1024,
163 + dtype=torch.float32).cuda(pynccl_comm.rank)
164 + with pynccl_comm.change_state(enable=True):
165 + if pynccl_comm.rank == 0:
166 + pynccl_comm.send(tensor)
167 + else:
168 + pynccl_comm.recv(tensor)
169 + result = tensor.mean().cpu().item()
170 + assert result == 1
171 +
172 +
173 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
174 + reason="Need at least 2 GPUs to run the test.")
175 +def test_pynccl_send_recv():
176 + distributed_run(send_recv_worker_fn, 2)
177 +
178 +
179 +@worker_fn_wrapper
180 +def multiple_send_recv_worker_fn():
181 + device = torch.device(f"cuda:{torch.distributed.get_rank()}")
182 + groups = [
183 + torch.distributed.new_group(ranks=[0, 2], backend="gloo"),
184 + torch.distributed.new_group(ranks=[1, 3], backend="gloo")
185 + ]
186 + group = groups[0] if torch.distributed.get_rank() in [0, 2] else groups[1]
187 + pynccl_comm = PyNcclCommunicator(group=group, device=device)
188 + if torch.distributed.get_rank() == 0:
189 + tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
190 + elif torch.distributed.get_rank() == 1:
191 + tensor = 2 * torch.ones(
192 + 16, 1024, 1024, dtype=torch.float32, device=device)
193 + else:
194 + tensor = torch.empty(16,
195 + 1024,
196 + 1024,
197 + dtype=torch.float32,
198 + device=device)
199 + with pynccl_comm.change_state(enable=True):
200 + if torch.distributed.get_rank() in [0, 1]:
201 + pynccl_comm.send(tensor)
202 + else:
203 + pynccl_comm.recv(tensor)
204 + result = tensor.mean().cpu().item()
205 + if torch.distributed.get_rank() in [0, 2]:
206 + assert result == 1
207 + else:
208 + assert result == 2
209 +
210 +
211 +@pytest.mark.skipif(torch.cuda.device_count() < 4,
212 + reason="Need at least 4 GPUs to run the test.")
213 +def test_pynccl_multiple_send_recv():
214 + distributed_run(multiple_send_recv_worker_fn, 4)
215 +
216 +
217 def test_ncclGetUniqueId():
218 lib = NCCLLibrary()
219 unique_id = lib.ncclGetUniqueId()
vllm/distributed/communication_op.py
+12 -6
@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
6 import torch
7 from torch.distributed import ProcessGroup
8
9 -from .parallel_state import (get_cpu_world_group,
9 +from .parallel_state import (get_cpu_world_group, get_pp_pynccl_communicator,
10 get_tensor_model_parallel_group,
11 get_tensor_model_parallel_rank,
12 get_tensor_model_parallel_world_size,
@@ -54,13 +54,19 @@ def graph_capture():
54 # graph, we use either custom all-reduce kernel or PyTorch NCCL.
55 # We always prioritize using custom all-reduce kernel but fall back
56 # to PyTorch or pynccl if it is disabled or not supported.
57 - pynccl_comm = get_tp_pynccl_communicator()
58 - if pynccl_comm is None:
59 - maybe_pynccl_context = nullcontext()
57 + tp_pynccl_comm = get_tp_pynccl_communicator()
58 + pp_pynccl_comm = get_pp_pynccl_communicator()
59 + if not tp_pynccl_comm:
60 + maybe_tp_pynccl_context = nullcontext()
61 else:
61 - maybe_pynccl_context = pynccl_comm.change_state(
62 + maybe_tp_pynccl_context = tp_pynccl_comm.change_state(
63 enable=True, stream=torch.cuda.current_stream())
63 - with maybe_pynccl_context:
64 + if not pp_pynccl_comm:
65 + maybe_pp_pynccl_context = nullcontext()
66 + else:
67 + maybe_pp_pynccl_context = pp_pynccl_comm.change_state(
68 + enable=True, stream=torch.cuda.current_stream())
69 + with maybe_tp_pynccl_context, maybe_pp_pynccl_context:
70 yield graph_capture_context
71
72
vllm/distributed/device_communicators/pynccl.py
+34
@@ -126,6 +126,40 @@ class PyNcclCommunicator:
126 ncclRedOpTypeEnum.from_torch(op), self.comm,
127 cudaStream_t(stream.cuda_stream))
128
129 + def send(self,
130 + tensor: torch.Tensor,
131 + dst: Optional[int] = None,
132 + stream=None):
133 + if self.disabled:
134 + return
135 + assert tensor.device == self.device, (
136 + f"this nccl communicator is created to work on {self.device}, "
137 + f"but the input tensor is on {tensor.device}")
138 + if stream is None:
139 + stream = self.stream
140 + if dst is None:
141 + dst = (self.rank + 1) % self.world_size
142 + self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
143 + ncclDataTypeEnum.from_torch(tensor.dtype), dst,
144 + self.comm, cudaStream_t(stream.cuda_stream))
145 +
146 + def recv(self,
147 + tensor: torch.Tensor,
148 + src: Optional[int] = None,
149 + stream=None):
150 + if self.disabled:
151 + return
152 + assert tensor.device == self.device, (
153 + f"this nccl communicator is created to work on {self.device}, "
154 + f"but the input tensor is on {tensor.device}")
155 + if stream is None:
156 + stream = self.stream
157 + if src is None:
158 + src = (self.rank - 1) % self.world_size
159 + self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
160 + ncclDataTypeEnum.from_torch(tensor.dtype), src,
161 + self.comm, cudaStream_t(stream.cuda_stream))
162 +
163 @contextmanager
164 def change_state(self,
165 enable: Optional[bool] = None,
vllm/distributed/device_communicators/pynccl_wrapper.py
+26
@@ -151,6 +151,22 @@ class NCCLLibrary:
151 ncclRedOp_t, ncclComm_t, cudaStream_t
152 ]),
153
154 + # ncclResult_t ncclSend(
155 + # const void* sendbuff, size_t count, ncclDataType_t datatype,
156 + # int dest, ncclComm_t comm, cudaStream_t stream);
157 + Function("ncclSend", ncclResult_t, [
158 + buffer_type, ctypes.c_size_t, ncclDataType_t, ctypes.c_int,
159 + ncclComm_t, cudaStream_t
160 + ]),
161 +
162 + # ncclResult_t ncclRecv(
163 + # void* recvbuff, size_t count, ncclDataType_t datatype,
164 + # int src, ncclComm_t comm, cudaStream_t stream);
165 + Function("ncclRecv", ncclResult_t, [
166 + buffer_type, ctypes.c_size_t, ncclDataType_t, ctypes.c_int,
167 + ncclComm_t, cudaStream_t
168 + ]),
169 +
170 # be cautious! this is a collective call, it will block until all
171 # processes in the communicator have called this function.
172 # because Python object destruction can happen in random order,
@@ -248,6 +264,16 @@ class NCCLLibrary:
264 datatype, op, comm,
265 stream))
266
267 + def ncclSend(self, sendbuff: buffer_type, count: int, datatype: int,
268 + dest: int, comm: ncclComm_t, stream: cudaStream_t) -> None:
269 + self.NCCL_CHECK(self._funcs["ncclSend"](sendbuff, count, datatype,
270 + dest, comm, stream))
271 +
272 + def ncclRecv(self, recvbuff: buffer_type, count: int, datatype: int,
273 + src: int, comm: ncclComm_t, stream: cudaStream_t) -> None:
274 + self.NCCL_CHECK(self._funcs["ncclRecv"](recvbuff, count, datatype, src,
275 + comm, stream))
276 +
277 def ncclCommDestroy(self, comm: ncclComm_t) -> None:
278 self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
279
vllm/distributed/parallel_state.py
+29 -5
@@ -22,6 +22,8 @@ _TP_PYNCCL_COMMUNICATOR = None
22 _TP_CA_COMMUNICATOR = None
23 # Pipeline model parallel group that the current rank belongs to.
24 _PP_DEVICE_GROUP: Optional[ProcessGroup] = None
25 +_PP_CPU_GROUP: Optional[ProcessGroup] = None
26 +_PP_PYNCCL_COMMUNICATOR = None
27
28 # when people blindly call `torch.distributed.all_reduce` etc,
29 # it will use this group. It is initialized with the `backend`
@@ -55,6 +57,11 @@ def set_custom_all_reduce(enable: bool):
57 _ENABLE_CUSTOM_ALL_REDUCE = enable
58
59
60 +def get_pp_pynccl_communicator():
61 + global _PP_PYNCCL_COMMUNICATOR
62 + return _PP_PYNCCL_COMMUNICATOR
63 +
64 +
65 def get_tp_pynccl_communicator():
66 global _TP_PYNCCL_COMMUNICATOR
67 return _TP_PYNCCL_COMMUNICATOR
@@ -180,10 +187,11 @@ def initialize_model_parallel(
187 _TP_CPU_GROUP = cpu_group
188
189 from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
183 - _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
184 - group=_TP_CPU_GROUP,
185 - device=_LOCAL_RANK,
186 - )
190 + if tensor_model_parallel_size > 1:
191 + _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
192 + group=_TP_CPU_GROUP,
193 + device=_LOCAL_RANK,
194 + )
195
196 # Initialize a custom fast all-reduce implementation.
197 if _ENABLE_CUSTOM_ALL_REDUCE:
@@ -195,17 +203,26 @@ def initialize_model_parallel(
203 )
204
205 # Build the pipeline model-parallel groups.
198 - global _PP_DEVICE_GROUP
206 + global _PP_DEVICE_GROUP, _PP_CPU_GROUP
207 + global _PP_PYNCCL_COMMUNICATOR
208 global _PP_GLOBAL_RANKS
209 assert _PP_DEVICE_GROUP is None, (
210 "pipeline model parallel group is already initialized")
211 for i in range(num_pipeline_model_parallel_groups):
212 ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
213 group = torch.distributed.new_group(ranks, backend=backend)
214 + cpu_group = torch.distributed.new_group(ranks, backend="gloo")
215 if rank in ranks:
216 _PP_DEVICE_GROUP = group
217 + _PP_CPU_GROUP = cpu_group
218 _PP_GLOBAL_RANKS = ranks
219
220 + if pipeline_model_parallel_size > 1:
221 + _PP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
222 + group=_PP_CPU_GROUP,
223 + device=_LOCAL_RANK,
224 + )
225 +
226
227 def ensure_model_parallel_initialized(
228 tensor_model_parallel_size: int,
@@ -267,6 +284,13 @@ def get_pipeline_model_parallel_group():
284 return _PP_DEVICE_GROUP
285
286
287 +def get_pipeline_model_parallel_cpu_group():
288 + """Get the pipeline model parallel cpu group the caller rank belongs to."""
289 + assert _PP_CPU_GROUP is not None, (
290 + "pipeline model parallel cpu group is not initialized")
291 + return _PP_CPU_GROUP
292 +
293 +
294 def get_tensor_model_parallel_world_size():
295 """Return world size for the tensor model parallel group."""
296 return torch.distributed.get_world_size(