@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f9c28fd40
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix CustomAllreduce nvlink topology detection (#3974)

[Bugfix] Fix CustomAllreduce pcie nvlink topology detection (#3974) (#4159)

Adam Tilghman committed Apr 18, 2024 at 15:32 UTC 8f9c28fd40a9daa744d08cacb0bd5ac2247a97d1
1 file changed +4 -2
vllm/distributed/device_communicators/custom_all_reduce.py
+4 -2
@@ -145,8 +145,10 @@ def _is_full_nvlink(rank, world_size):
145 for i in range(world_size):
146 if i != rank:
147 try:
148 - link_state = pynvml.nvmlDeviceGetNvLinkState(handle, i)
149 - if not link_state:
148 + peer_handle = pynvml.nvmlDeviceGetHandleByIndex(i)
149 + p2p_status = pynvml.nvmlDeviceGetP2PStatus(
150 + handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
151 + if p2p_status != pynvml.NVML_P2P_STATUS_OK:
152 return False
153 except pynvml.NVMLError as error:
154 logger.info(