@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db5ec52ad7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[bugfix][distributed] improve p2p capability test (#5612)

[bugfix][distributed] do not error if two processes do not agree on p2p capability (#5612)

youkaichao committed Jun 18, 2024 at 00:21 UTC db5ec52ad7dc69dbe8dd9ba25fe8f2c6ce35a4cf
1 file changed +8 -2
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+8 -2
@@ -71,6 +71,7 @@ def consumer(batch_tgt: Sequence[int],
71 if open_success:
72 # modify the memory
73 lib.cudaMemset(pointer, 2, 1024)
74 + lib.cudaDeviceSynchronize()
75 # use two queues to simulate barrier
76 producer_queue.get()
77 consumer_queue.put(0)
@@ -142,8 +143,13 @@ def can_actually_p2p(
143 for src, tgt in zip(batch_src, batch_tgt):
144 a = result_queue.get()
145 b = result_queue.get()
145 - assert a == b
146 - result.append(a)
146 + if a != b:
147 + logger.warning(
148 + "Two processes do not agree on the P2P access"
149 + " status on %d -> %d, treat as disabled.", src, tgt)
150 + result.append(False)
151 + else:
152 + result.append(a)
153 return result
154
155