A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Add synchronize to prevent possible data race (#6788)
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Tyler Michael Smith committed
Jul 25, 2024 at 13:40 UTC
95db75de64bec34f4d80acff92c62d1cdfa94688
1 file changed
+7
vllm/distributed/parallel_state.py
+7
@@ -243,6 +243,13 @@ class GroupCoordinator:
243
ca_comm = self.ca_comm
244
maybe_ca_context = nullcontext(
245
) if ca_comm is None else ca_comm.capture()
246
+
247
+ # ensure all initialization operations complete before attempting to
248
+ # capture the graph on another stream
249
+ curr_stream = torch.cuda.current_stream()
250
+ if curr_stream != stream:
251
+ stream.wait_stream(curr_stream)
252
+
253
with torch.cuda.stream(stream), maybe_ca_context:
254
# In graph mode, we have to be very careful about the collective
255
# operations. The current status is: