A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] avoid too many cuda context by caching p2p test (#4021)
youkaichao committed
Apr 13, 2024 at 23:40 UTC
2cd6b4f3625466eb5849bcfd7a6fb316735adab8
3 files changed
+116
-33
vllm/distributed/device_communicators/custom_all_reduce.py
+21
-32
@@ -42,12 +42,17 @@ def init_custom_ar() -> None:
42
" disable_custom_all_reduce=True explicitly.", world_size,
43
str(_SUPPORTED_WORLD_SIZES))
44
return
45
- if not _can_p2p(rank, world_size):
45
+ num_dev = torch.cuda.device_count()
46
+ # note: num dev can be larger than world_size if we're only using
47
+ # first few GPUs
48
+ if num_dev < world_size:
49
logger.warn(
47
- "Custom allreduce is disabled because your platform lacks GPU P2P"
48
- " capability or P2P test failed. To silence this warning, specify"
49
- " disable_custom_all_reduce=True explicitly.")
50
- return
50
+ "Cannot test GPU P2P because not all GPUs are visible to the "
51
+ "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
52
+ " is set.")
53
+ return False
54
+ # test nvlink first, this will filter out most of the cases
55
+ # where custom allreduce is not supported
56
full_nvlink = _is_full_nvlink(rank, world_size)
57
if world_size > 2 and not full_nvlink:
58
logger.warn(
@@ -55,6 +60,15 @@ def init_custom_ar() -> None:
60
" than two PCIe-only GPUs. To silence this warning, specify"
61
" disable_custom_all_reduce=True explicitly.")
62
return
63
+ # test P2P capability
64
+ # this is expensive to compute at the first time
65
+ # then we cache the result
66
+ if not _can_p2p(rank, world_size):
67
+ logger.warn(
68
+ "Custom allreduce is disabled because your platform lacks GPU P2P"
69
+ " capability or P2P test failed. To silence this warning, specify"
70
+ " disable_custom_all_reduce=True explicitly.")
71
+ return
72
_CA_HANDLE = CustomAllreduce(rank, world_size, full_nvlink)
73
74
@@ -143,40 +157,15 @@ def _is_full_nvlink(rank, world_size):
157
158
159
def _can_p2p(rank: int, world_size: int) -> bool:
146
- num_dev = torch.cuda.device_count()
147
- # note: num dev can be larger than world_size if we're only using
148
- # first few GPUs
149
- if num_dev < world_size:
150
- logger.warn(
151
- "Cannot test GPU P2P because not all GPUs are visible to the "
152
- "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
153
- " is set.")
154
- return False
160
+ from vllm.distributed.utils import gpu_p2p_access_check
161
for i in range(world_size):
162
if i == rank:
163
continue
158
- if not torch.cuda.can_device_access_peer(rank, i):
159
- return False
160
- # on some platforms, P2P support might be buggy and we need
161
- # additional checks. See also:
162
- # https://github.com/vllm-project/vllm/issues/2728
163
- if not _can_actually_p2p(rank, i):
164
+ if not gpu_p2p_access_check(rank, i):
165
return False
166
return True
167
168
168
-# code partly borrowed from
169
-# https://github.com/turboderp/exllamav2/blob/1c67f97f3d2a968605a9c31ab791a05c85bb7879/exllamav2/compat.py#L10
170
-# License: MIT
171
-def _can_actually_p2p(idx_a, idx_b):
172
- dev_i = f"cuda:{idx_a}"
173
- dev_j = f"cuda:{idx_b}"
174
- a = torch.randn(5, device=dev_i) + 123.0
175
- b = a.to(dev_j)
176
- c = b.to(dev_i)
177
- return torch.all(a == c)
178
-
179
-
169
class CustomAllreduce:
170
171
# max_size: max supported allreduce size
vllm/distributed/parallel_state.py
+9
@@ -41,6 +41,13 @@ _CPU_WORLD_GROUP = None
41
# source rank when broadcasting from the first or last pipeline stage.
42
_PIPELINE_GLOBAL_RANKS = None
43
44
+_LOCAL_RANK = -1
45
+
46
+
47
+def get_local_rank():
48
+ global _LOCAL_RANK
49
+ return _LOCAL_RANK
50
+
51
52
def init_distributed_environment(
53
world_size: int = -1,
@@ -66,6 +73,8 @@ def init_distributed_environment(
73
ranks = list(range(torch.distributed.get_world_size()))
74
_CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
75
backend="gloo")
76
+ global _LOCAL_RANK
77
+ _LOCAL_RANK = local_rank
78
79
80
def initialize_model_parallel(
vllm/distributed/utils.py
+86
-1
@@ -2,9 +2,18 @@
2
# Adapted from
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
-from typing import Sequence
5
+import json
6
+import os
7
+from typing import Dict, Optional, Sequence
8
9
import torch
10
+import torch.distributed as dist
11
+
12
+from vllm.logger import init_logger
13
+
14
+from .parallel_state import get_cpu_world_group, get_local_rank
15
+
16
+logger = init_logger(__name__)
17
18
19
def ensure_divisibility(numerator, denominator):
@@ -46,3 +55,79 @@ def split_tensor_along_last_dim(
55
return tuple(chunk.contiguous() for chunk in tensor_list)
56
57
return tensor_list
58
+
59
+
60
+# code partly borrowed from
61
+# https://github.com/turboderp/exllamav2/blob/1c67f97f3d2a968605a9c31ab791a05c85bb7879/exllamav2/compat.py#L10
62
+# License: MIT
63
+def _can_actually_p2p(idx_a, idx_b):
64
+ dev_i = f"cuda:{idx_a}"
65
+ dev_j = f"cuda:{idx_b}"
66
+ a = torch.randn(5, device=dev_i) + 123.0
67
+ b = a.to(dev_j)
68
+ c = b.to(dev_i)
69
+ return torch.all(a == c).cpu().item()
70
+
71
+
72
+# why do we need this cache?
73
+# 1. we can have runtime checks for P2P access, where every process checks
74
+# P2P access to all other GPUs. Unfortunately, the test might cost many
75
+# (world_size * world_size) cuda context, and reduce the memory available
76
+# for the model. see https://github.com/vllm-project/vllm/issues/3821
77
+# 2. alternatively, we can have a p2p map that is generated by the master
78
+# process and broadcasted to all other processes. This still requires
79
+# #world_size of cuda context, belonging to the master process, on each GPU.
80
+# 3. we can have a cache file, that records the p2p access status. The first
81
+# time the master process checks the p2p access, it will generate the cache
82
+# file, at the cost of #world_size of cuda context. Later on, all processes
83
+# can read the cache file to check the p2p access status without any cost of
84
+# additional cuda context.
85
+# Note that the cache file is suffixed by the CUDA_VISIBLE_DEVICES, so that we
86
+# can have different cache files for different CUDA_VISIBLE_DEVICES settings,
87
+# e.g. used by different vllm engines. The device id in the cache file is a
88
+# **local** device id, i.e. from 0 to num_dev-1, where num_dev is the number
89
+# of visible devices in the vllm engine.
90
+_gpu_p2p_access_cache: Optional[Dict[str, bool]] = None
91
+
92
+
93
+def gpu_p2p_access_check(i: int, j: int) -> bool:
94
+ """Check if GPU i can access GPU j."""
95
+
96
+ # if the cache variable is already calculated,
97
+ # read from the cache instead of checking it again
98
+ global _gpu_p2p_access_cache
99
+ if _gpu_p2p_access_cache is not None:
100
+ return _gpu_p2p_access_cache[f"{i}->{j}"]
101
+
102
+ is_distributed = dist.is_initialized()
103
+
104
+ num_dev = torch.cuda.device_count()
105
+ cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None)
106
+ if cuda_visible_devices is None:
107
+ cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
108
+ path = os.path.expanduser(
109
+ f"~/.config/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json")
110
+ os.makedirs(os.path.dirname(path), exist_ok=True)
111
+ if (not is_distributed or get_local_rank() == 0) \
112
+ and (not os.path.exists(path)):
113
+ # only the local master process (with local_rank == 0) can
114
+ # enter this block to calculate the cache
115
+ logger.info(f"generating GPU P2P access cache for in {path}")
116
+ cache = {}
117
+ for _i in range(num_dev):
118
+ for _j in range(num_dev):
119
+ # on some platforms, P2P support might be buggy and we need
120
+ # additional checks. See also:
121
+ # https://github.com/vllm-project/vllm/issues/2728
122
+ cache[f"{_i}->{_j}"] = torch.cuda.can_device_access_peer(
123
+ _i, _j) and _can_actually_p2p(_i, _j)
124
+ with open(path, "w") as f:
125
+ json.dump(cache, f, indent=4)
126
+ if is_distributed:
127
+ cpu_world_group = get_cpu_world_group()
128
+ dist.barrier(cpu_world_group)
129
+ logger.info(f"reading GPU P2P access cache from {path}")
130
+ with open(path, "r") as f:
131
+ cache = json.load(f)
132
+ _gpu_p2p_access_cache = cache
133
+ return _gpu_p2p_access_cache[f"{i}->{j}"]