A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[distributed] remove pynccl's redundant stream (#11744)
cennn committed
Jan 5, 2025 at 23:09 UTC
635b897246da121238454ed4b2bbc87cb4d4166b
3 files changed
+12
-24
tests/distributed/test_pynccl.py
+2
-3
@@ -137,9 +137,8 @@ def worker_fn_with_cudagraph():
137
# run something in the default stream to initialize torch engine
138
a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
139
torch.cuda.synchronize()
140
- with torch.cuda.graph(
141
- graph, stream=pynccl_comm.stream), pynccl_comm.change_state(
142
- enable=True):
140
+ with torch.cuda.graph(graph), \
141
+ pynccl_comm.change_state(enable=True):
142
a_out = pynccl_comm.all_reduce(a)
143
torch.cuda.synchronize()
144
graph.replay()
vllm/distributed/device_communicators/pynccl.py
+9
-19
@@ -51,7 +51,6 @@ class PyNcclCommunicator:
51
if self.world_size == 1:
52
self.available = False
53
self.disabled = True
54
- self.stream = None
54
return
55
try:
56
self.nccl = NCCLLibrary(library_path)
@@ -60,7 +59,6 @@ class PyNcclCommunicator:
59
# e.g. in a non-GPU environment
60
self.available = False
61
self.disabled = True
63
- self.stream = None
62
return
63
64
self.available = True
@@ -98,12 +96,12 @@ class PyNcclCommunicator:
96
with torch.cuda.device(device):
97
self.comm: ncclComm_t = self.nccl.ncclCommInitRank(
98
self.world_size, self.unique_id, self.rank)
101
- self.stream = torch.cuda.Stream()
99
100
+ stream = torch.cuda.current_stream()
101
# A small all_reduce for warmup.
102
data = torch.zeros(1, device=device)
103
self.all_reduce(data)
106
- self.stream.synchronize()
104
+ stream.synchronize()
105
del data
106
107
def all_reduce(self,
@@ -122,7 +120,7 @@ class PyNcclCommunicator:
120
out_tensor = torch.empty_like(in_tensor)
121
122
if stream is None:
125
- stream = self.stream
123
+ stream = torch.cuda.current_stream()
124
self.nccl.ncclAllReduce(buffer_type(in_tensor.data_ptr()),
125
buffer_type(out_tensor.data_ptr()),
126
in_tensor.numel(),
@@ -144,7 +142,7 @@ class PyNcclCommunicator:
142
f"this nccl communicator is created to work on {self.device}, "
143
f"but the input tensor is on {input_tensor.device}")
144
if stream is None:
147
- stream = self.stream
145
+ stream = torch.cuda.current_stream()
146
self.nccl.ncclAllGather(
147
buffer_type(input_tensor.data_ptr()),
148
buffer_type(output_tensor.data_ptr()), input_tensor.numel(),
@@ -165,7 +163,7 @@ class PyNcclCommunicator:
163
f"this nccl communicator is created to work on {self.device}, "
164
f"but the input tensor is on {input_tensor.device}")
165
if stream is None:
168
- stream = self.stream
166
+ stream = torch.cuda.current_stream()
167
self.nccl.ncclReduceScatter(
168
buffer_type(input_tensor.data_ptr()),
169
buffer_type(output_tensor.data_ptr()), output_tensor.numel(),
@@ -180,7 +178,7 @@ class PyNcclCommunicator:
178
f"this nccl communicator is created to work on {self.device}, "
179
f"but the input tensor is on {tensor.device}")
180
if stream is None:
183
- stream = self.stream
181
+ stream = torch.cuda.current_stream()
182
self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
183
ncclDataTypeEnum.from_torch(tensor.dtype), dst,
184
self.comm, cudaStream_t(stream.cuda_stream))
@@ -192,7 +190,7 @@ class PyNcclCommunicator:
190
f"this nccl communicator is created to work on {self.device}, "
191
f"but the input tensor is on {tensor.device}")
192
if stream is None:
195
- stream = self.stream
193
+ stream = torch.cuda.current_stream()
194
self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
195
ncclDataTypeEnum.from_torch(tensor.dtype), src,
196
self.comm, cudaStream_t(stream.cuda_stream))
@@ -204,7 +202,7 @@ class PyNcclCommunicator:
202
f"this nccl communicator is created to work on {self.device}, "
203
f"but the input tensor is on {tensor.device}")
204
if stream is None:
207
- stream = self.stream
205
+ stream = torch.cuda.current_stream()
206
if src == self.rank:
207
sendbuff = buffer_type(tensor.data_ptr())
208
# NCCL requires the sender also to have a receive buffer
@@ -217,9 +215,7 @@ class PyNcclCommunicator:
215
self.comm, cudaStream_t(stream.cuda_stream))
216
217
@contextmanager
220
- def change_state(self,
221
- enable: Optional[bool] = None,
222
- stream: Optional[torch.cuda.Stream] = None):
218
+ def change_state(self, enable: Optional[bool] = None):
219
"""
220
A context manager to change the state of the communicator.
221
"""
@@ -227,15 +223,9 @@ class PyNcclCommunicator:
223
# guess a default value when not specified
224
enable = self.available
225
230
- if stream is None:
231
- stream = self.stream
232
-
226
old_disable = self.disabled
234
- old_stream = self.stream
227
236
- self.stream = stream
228
self.disabled = not enable
229
yield
230
231
self.disabled = old_disable
241
- self.stream = old_stream
vllm/distributed/parallel_state.py
+1
-2
@@ -310,8 +310,7 @@ class GroupCoordinator:
310
if not pynccl_comm:
311
maybe_pynccl_context = nullcontext()
312
else:
313
- maybe_pynccl_context = pynccl_comm.change_state(
314
- stream=torch.cuda.current_stream())
313
+ maybe_pynccl_context = pynccl_comm.change_state()
314
with maybe_pynccl_context:
315
yield graph_capture_context
316