@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 639159b2a6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[distributed][misc] add specialized method for cuda platform (#7249)

youkaichao committed Aug 7, 2024 at 08:54 UTC 639159b2a665173bc7a81625887b8e76f85e2e32
3 files changed +42 -53
vllm/distributed/device_communicators/custom_all_reduce.py
+6 -2
@@ -11,7 +11,8 @@ from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11 gpu_p2p_access_check)
12 from vllm.distributed.parallel_state import in_the_same_node_as
13 from vllm.logger import init_logger
14 -from vllm.utils import cuda_device_count_stateless, is_full_nvlink
14 +from vllm.platforms import current_platform
15 +from vllm.utils import cuda_device_count_stateless
16
17 try:
18 assert ops.is_custom_op_supported("_C_custom_ar::meta_size")
@@ -113,7 +114,10 @@ class CustomAllreduce:
114 # test nvlink first, this will filter out most of the cases
115 # where custom allreduce is not supported
116 # this checks hardware and driver support for NVLink
116 - full_nvlink = is_full_nvlink(physical_device_ids)
117 + assert current_platform.is_cuda()
118 + from vllm.platforms.cuda import CudaPlatform
119 + cuda_platform: CudaPlatform = current_platform
120 + full_nvlink = cuda_platform.is_full_nvlink(physical_device_ids)
121 if world_size > 2 and not full_nvlink:
122 logger.warning(
123 "Custom allreduce is disabled because it's not supported on"
vllm/platforms/cuda.py
+36 -1
@@ -4,12 +4,21 @@ pynvml. However, it should not initialize cuda context.
4
5 import os
6 from functools import lru_cache, wraps
7 -from typing import Tuple
7 +from typing import List, Tuple
8
9 import pynvml
10
11 +from vllm.logger import init_logger
12 +
13 from .interface import Platform, PlatformEnum
14
15 +logger = init_logger(__name__)
16 +
17 +# NVML utils
18 +# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
19 +# all the related functions work on real physical device ids.
20 +# the major benefit of using NVML is that it will not initialize CUDA
21 +
22
23 def with_nvml_context(fn):
24
@@ -47,3 +56,29 @@ class CudaPlatform(Platform):
56 def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
57 physical_device_id = device_id_to_physical_device_id(device_id)
58 return get_physical_device_capability(physical_device_id)
59 +
60 + @staticmethod
61 + @with_nvml_context
62 + def is_full_nvlink(physical_device_ids: List[int]) -> bool:
63 + """
64 + query if the set of gpus are fully connected by nvlink (1 hop)
65 + """
66 + handles = [
67 + pynvml.nvmlDeviceGetHandleByIndex(i) for i in physical_device_ids
68 + ]
69 + for i, handle in enumerate(handles):
70 + for j, peer_handle in enumerate(handles):
71 + if i < j:
72 + try:
73 + p2p_status = pynvml.nvmlDeviceGetP2PStatus(
74 + handle, peer_handle,
75 + pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
76 + if p2p_status != pynvml.NVML_P2P_STATUS_OK:
77 + return False
78 + except pynvml.NVMLError as error:
79 + logger.error(
80 + "NVLink detection failed. This is normal if your"
81 + " machine has no NVLink equipped.",
82 + exc_info=error)
83 + return False
84 + return True
vllm/utils.py
-50
@@ -1034,56 +1034,6 @@ def cuda_device_count_stateless() -> int:
1034 return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
1035
1036
1037 -# NVML utils
1038 -# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
1039 -# all the related functions work on real physical device ids.
1040 -# the major benefit of using NVML is that it will not initialize CUDA
1041 -
1042 -try:
1043 - import pynvml
1044 -except ImportError:
1045 - # For non-NV devices
1046 - pynvml = None
1047 -
1048 -
1049 -def with_nvml_context(fn):
1050 -
1051 - @wraps(fn)
1052 - def wrapper(*args, **kwargs):
1053 - if pynvml is not None:
1054 - pynvml.nvmlInit()
1055 - try:
1056 - return fn(*args, **kwargs)
1057 - finally:
1058 - if pynvml is not None:
1059 - pynvml.nvmlShutdown()
1060 -
1061 - return wrapper
1062 -
1063 -
1064 -@with_nvml_context
1065 -def is_full_nvlink(device_ids: List[int]) -> bool:
1066 - """
1067 - query if the set of gpus are fully connected by nvlink (1 hop)
1068 - """
1069 - handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in device_ids]
1070 - for i, handle in enumerate(handles):
1071 - for j, peer_handle in enumerate(handles):
1072 - if i < j:
1073 - try:
1074 - p2p_status = pynvml.nvmlDeviceGetP2PStatus(
1075 - handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
1076 - if p2p_status != pynvml.NVML_P2P_STATUS_OK:
1077 - return False
1078 - except pynvml.NVMLError as error:
1079 - logger.error(
1080 - "NVLink detection failed. This is normal if your"
1081 - " machine has no NVLink equipped.",
1082 - exc_info=error)
1083 - return False
1084 - return True
1085 -
1086 -
1037 #From: https://stackoverflow.com/a/4104188/2749989
1038 def run_once(f):
1039