A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[bugfix][distributed] improve p2p capability test (#5612)
[bugfix][distributed] do not error if two processes do not agree on p2p capability (#5612)
youkaichao committed
Jun 18, 2024 at 00:21 UTC
db5ec52ad7dc69dbe8dd9ba25fe8f2c6ce35a4cf
1 file changed
+8
-2
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+8
-2
@@ -71,6 +71,7 @@ def consumer(batch_tgt: Sequence[int],
71
if open_success:
72
# modify the memory
73
lib.cudaMemset(pointer, 2, 1024)
74
+ lib.cudaDeviceSynchronize()
75
# use two queues to simulate barrier
76
producer_queue.get()
77
consumer_queue.put(0)
@@ -142,8 +143,13 @@ def can_actually_p2p(
143
for src, tgt in zip(batch_src, batch_tgt):
144
a = result_queue.get()
145
b = result_queue.get()
145
- assert a == b
146
- result.append(a)
146
+ if a != b:
147
+ logger.warning(
148
+ "Two processes do not agree on the P2P access"
149
+ " status on %d -> %d, treat as disabled.", src, tgt)
150
+ result.append(False)
151
+ else:
152
+ result.append(a)
153
return result
154
155