A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix CustomAllreduce nvlink topology detection (#3974)
[Bugfix] Fix CustomAllreduce pcie nvlink topology detection (#3974) (#4159)
Adam Tilghman committed
Apr 18, 2024 at 15:32 UTC
8f9c28fd40a9daa744d08cacb0bd5ac2247a97d1
1 file changed
+4
-2
vllm/distributed/device_communicators/custom_all_reduce.py
+4
-2
@@ -145,8 +145,10 @@ def _is_full_nvlink(rank, world_size):
145
for i in range(world_size):
146
if i != rank:
147
try:
148
- link_state = pynvml.nvmlDeviceGetNvLinkState(handle, i)
149
- if not link_state:
148
+ peer_handle = pynvml.nvmlDeviceGetHandleByIndex(i)
149
+ p2p_status = pynvml.nvmlDeviceGetP2PStatus(
150
+ handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
151
+ if p2p_status != pynvml.NVML_P2P_STATUS_OK:
152
return False
153
except pynvml.NVMLError as error:
154
logger.info(