A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[distributed][misc] add specialized method for cuda platform (#7249)
youkaichao committed
Aug 7, 2024 at 08:54 UTC
639159b2a665173bc7a81625887b8e76f85e2e32
3 files changed
+42
-53
vllm/distributed/device_communicators/custom_all_reduce.py
+6
-2
@@ -11,7 +11,8 @@ from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11
gpu_p2p_access_check)
12
from vllm.distributed.parallel_state import in_the_same_node_as
13
from vllm.logger import init_logger
14
-from vllm.utils import cuda_device_count_stateless, is_full_nvlink
14
+from vllm.platforms import current_platform
15
+from vllm.utils import cuda_device_count_stateless
16
17
try:
18
assert ops.is_custom_op_supported("_C_custom_ar::meta_size")
@@ -113,7 +114,10 @@ class CustomAllreduce:
114
# test nvlink first, this will filter out most of the cases
115
# where custom allreduce is not supported
116
# this checks hardware and driver support for NVLink
116
- full_nvlink = is_full_nvlink(physical_device_ids)
117
+ assert current_platform.is_cuda()
118
+ from vllm.platforms.cuda import CudaPlatform
119
+ cuda_platform: CudaPlatform = current_platform
120
+ full_nvlink = cuda_platform.is_full_nvlink(physical_device_ids)
121
if world_size > 2 and not full_nvlink:
122
logger.warning(
123
"Custom allreduce is disabled because it's not supported on"
vllm/platforms/cuda.py
+36
-1
@@ -4,12 +4,21 @@ pynvml. However, it should not initialize cuda context.
4
5
import os
6
from functools import lru_cache, wraps
7
-from typing import Tuple
7
+from typing import List, Tuple
8
9
import pynvml
10
11
+from vllm.logger import init_logger
12
+
13
from .interface import Platform, PlatformEnum
14
15
+logger = init_logger(__name__)
16
+
17
+# NVML utils
18
+# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
19
+# all the related functions work on real physical device ids.
20
+# the major benefit of using NVML is that it will not initialize CUDA
21
+
22
23
def with_nvml_context(fn):
24
@@ -47,3 +56,29 @@ class CudaPlatform(Platform):
56
def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
57
physical_device_id = device_id_to_physical_device_id(device_id)
58
return get_physical_device_capability(physical_device_id)
59
+
60
+ @staticmethod
61
+ @with_nvml_context
62
+ def is_full_nvlink(physical_device_ids: List[int]) -> bool:
63
+ """
64
+ query if the set of gpus are fully connected by nvlink (1 hop)
65
+ """
66
+ handles = [
67
+ pynvml.nvmlDeviceGetHandleByIndex(i) for i in physical_device_ids
68
+ ]
69
+ for i, handle in enumerate(handles):
70
+ for j, peer_handle in enumerate(handles):
71
+ if i < j:
72
+ try:
73
+ p2p_status = pynvml.nvmlDeviceGetP2PStatus(
74
+ handle, peer_handle,
75
+ pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
76
+ if p2p_status != pynvml.NVML_P2P_STATUS_OK:
77
+ return False
78
+ except pynvml.NVMLError as error:
79
+ logger.error(
80
+ "NVLink detection failed. This is normal if your"
81
+ " machine has no NVLink equipped.",
82
+ exc_info=error)
83
+ return False
84
+ return True
vllm/utils.py
-50
@@ -1034,56 +1034,6 @@ def cuda_device_count_stateless() -> int:
1034
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
1035
1036
1037
-# NVML utils
1038
-# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
1039
-# all the related functions work on real physical device ids.
1040
-# the major benefit of using NVML is that it will not initialize CUDA
1041
-
1042
-try:
1043
- import pynvml
1044
-except ImportError:
1045
- # For non-NV devices
1046
- pynvml = None
1047
-
1048
-
1049
-def with_nvml_context(fn):
1050
-
1051
- @wraps(fn)
1052
- def wrapper(*args, **kwargs):
1053
- if pynvml is not None:
1054
- pynvml.nvmlInit()
1055
- try:
1056
- return fn(*args, **kwargs)
1057
- finally:
1058
- if pynvml is not None:
1059
- pynvml.nvmlShutdown()
1060
-
1061
- return wrapper
1062
-
1063
-
1064
-@with_nvml_context
1065
-def is_full_nvlink(device_ids: List[int]) -> bool:
1066
- """
1067
- query if the set of gpus are fully connected by nvlink (1 hop)
1068
- """
1069
- handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in device_ids]
1070
- for i, handle in enumerate(handles):
1071
- for j, peer_handle in enumerate(handles):
1072
- if i < j:
1073
- try:
1074
- p2p_status = pynvml.nvmlDeviceGetP2PStatus(
1075
- handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
1076
- if p2p_status != pynvml.NVML_P2P_STATUS_OK:
1077
- return False
1078
- except pynvml.NVMLError as error:
1079
- logger.error(
1080
- "NVLink detection failed. This is normal if your"
1081
- " machine has no NVLink equipped.",
1082
- exc_info=error)
1083
- return False
1084
- return True
1085
-
1086
-
1037
#From: https://stackoverflow.com/a/4104188/2749989
1038
def run_once(f):
1039