A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Distributed] fix _is_full_nvlink detection (#4233)
youkaichao committed
Apr 21, 2024 at 23:04 UTC
747b1a7147515c08491ef4aa1b23ea23329966ed
1 file changed
+30
-18
vllm/distributed/device_communicators/custom_all_reduce.py
+30
-18
@@ -1,5 +1,6 @@
1
+import os
2
from contextlib import contextmanager
2
-from typing import Optional
3
+from typing import List, Optional
4
5
import torch
6
import torch.distributed as dist
@@ -53,14 +54,20 @@ def init_custom_ar() -> None:
54
return False
55
# test nvlink first, this will filter out most of the cases
56
# where custom allreduce is not supported
56
- full_nvlink = _is_full_nvlink(rank, world_size)
57
+ if "CUDA_VISIBLE_DEVICES" in os.environ:
58
+ device_ids = list(
59
+ map(int, os.environ["CUDA_VISIBLE_DEVICES"].split(",")))
60
+ else:
61
+ device_ids = list(range(num_dev))
62
+ # this checks hardware and driver support for NVLink
63
+ full_nvlink = _is_full_nvlink(device_ids)
64
if world_size > 2 and not full_nvlink:
65
logger.warn(
66
"Custom allreduce is disabled because it's not supported on more"
67
" than two PCIe-only GPUs. To silence this warning, specify"
68
" disable_custom_all_reduce=True explicitly.")
69
return
63
- # test P2P capability
70
+ # test P2P capability, this checks software/cudaruntime support
71
# this is expensive to compute at the first time
72
# then we cache the result
73
if not _can_p2p(rank, world_size):
@@ -138,23 +145,28 @@ def _nvml():
145
pynvml.nvmlShutdown()
146
147
141
-# query if the set of gpus are fully connected by nvlink (1 hop)
148
@_nvml()
143
-def _is_full_nvlink(rank, world_size):
144
- handle = pynvml.nvmlDeviceGetHandleByIndex(rank)
145
- for i in range(world_size):
146
- if i != rank:
147
- try:
148
- peer_handle = pynvml.nvmlDeviceGetHandleByIndex(i)
149
- p2p_status = pynvml.nvmlDeviceGetP2PStatus(
150
- handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
151
- if p2p_status != pynvml.NVML_P2P_STATUS_OK:
149
+def _is_full_nvlink(device_ids: List[int]) -> bool:
150
+ """
151
+ query if the set of gpus are fully connected by nvlink (1 hop)
152
+ Note that `pynvml` is not affected by `CUDA_VISIBLE_DEVICES`,
153
+ so it works on real physical device ids.
154
+ """
155
+ handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in device_ids]
156
+ for i, handle in enumerate(handles):
157
+ for j, peer_handle in enumerate(handles):
158
+ if i < j:
159
+ try:
160
+ p2p_status = pynvml.nvmlDeviceGetP2PStatus(
161
+ handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
162
+ if p2p_status != pynvml.NVML_P2P_STATUS_OK:
163
+ return False
164
+ except pynvml.NVMLError as error:
165
+ logger.error(
166
+ "NVLink detection failed. This is normal if your"
167
+ " machine has no NVLink equipped.",
168
+ exc_info=error)
169
return False
153
- except pynvml.NVMLError as error:
154
- logger.info(
155
- f"NVLink detection failed with message \"{str(error)}\". "
156
- "This is normal if your machine has no NVLink equipped")
157
- return False
170
return True
171
172