@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2cd6b4f362
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] avoid too many cuda context by caching p2p test (#4021)

youkaichao committed Apr 13, 2024 at 23:40 UTC 2cd6b4f3625466eb5849bcfd7a6fb316735adab8
3 files changed +116 -33
vllm/distributed/device_communicators/custom_all_reduce.py
+21 -32
@@ -42,12 +42,17 @@ def init_custom_ar() -> None:
42 " disable_custom_all_reduce=True explicitly.", world_size,
43 str(_SUPPORTED_WORLD_SIZES))
44 return
45 - if not _can_p2p(rank, world_size):
45 + num_dev = torch.cuda.device_count()
46 + # note: num dev can be larger than world_size if we're only using
47 + # first few GPUs
48 + if num_dev < world_size:
49 logger.warn(
47 - "Custom allreduce is disabled because your platform lacks GPU P2P"
48 - " capability or P2P test failed. To silence this warning, specify"
49 - " disable_custom_all_reduce=True explicitly.")
50 - return
50 + "Cannot test GPU P2P because not all GPUs are visible to the "
51 + "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
52 + " is set.")
53 + return False
54 + # test nvlink first, this will filter out most of the cases
55 + # where custom allreduce is not supported
56 full_nvlink = _is_full_nvlink(rank, world_size)
57 if world_size > 2 and not full_nvlink:
58 logger.warn(
@@ -55,6 +60,15 @@ def init_custom_ar() -> None:
60 " than two PCIe-only GPUs. To silence this warning, specify"
61 " disable_custom_all_reduce=True explicitly.")
62 return
63 + # test P2P capability
64 + # this is expensive to compute at the first time
65 + # then we cache the result
66 + if not _can_p2p(rank, world_size):
67 + logger.warn(
68 + "Custom allreduce is disabled because your platform lacks GPU P2P"
69 + " capability or P2P test failed. To silence this warning, specify"
70 + " disable_custom_all_reduce=True explicitly.")
71 + return
72 _CA_HANDLE = CustomAllreduce(rank, world_size, full_nvlink)
73
74
@@ -143,40 +157,15 @@ def _is_full_nvlink(rank, world_size):
157
158
159 def _can_p2p(rank: int, world_size: int) -> bool:
146 - num_dev = torch.cuda.device_count()
147 - # note: num dev can be larger than world_size if we're only using
148 - # first few GPUs
149 - if num_dev < world_size:
150 - logger.warn(
151 - "Cannot test GPU P2P because not all GPUs are visible to the "
152 - "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
153 - " is set.")
154 - return False
160 + from vllm.distributed.utils import gpu_p2p_access_check
161 for i in range(world_size):
162 if i == rank:
163 continue
158 - if not torch.cuda.can_device_access_peer(rank, i):
159 - return False
160 - # on some platforms, P2P support might be buggy and we need
161 - # additional checks. See also:
162 - # https://github.com/vllm-project/vllm/issues/2728
163 - if not _can_actually_p2p(rank, i):
164 + if not gpu_p2p_access_check(rank, i):
165 return False
166 return True
167
168
168 -# code partly borrowed from
169 -# https://github.com/turboderp/exllamav2/blob/1c67f97f3d2a968605a9c31ab791a05c85bb7879/exllamav2/compat.py#L10
170 -# License: MIT
171 -def _can_actually_p2p(idx_a, idx_b):
172 - dev_i = f"cuda:{idx_a}"
173 - dev_j = f"cuda:{idx_b}"
174 - a = torch.randn(5, device=dev_i) + 123.0
175 - b = a.to(dev_j)
176 - c = b.to(dev_i)
177 - return torch.all(a == c)
178 -
179 -
169 class CustomAllreduce:
170
171 # max_size: max supported allreduce size
vllm/distributed/parallel_state.py
+9
@@ -41,6 +41,13 @@ _CPU_WORLD_GROUP = None
41 # source rank when broadcasting from the first or last pipeline stage.
42 _PIPELINE_GLOBAL_RANKS = None
43
44 +_LOCAL_RANK = -1
45 +
46 +
47 +def get_local_rank():
48 + global _LOCAL_RANK
49 + return _LOCAL_RANK
50 +
51
52 def init_distributed_environment(
53 world_size: int = -1,
@@ -66,6 +73,8 @@ def init_distributed_environment(
73 ranks = list(range(torch.distributed.get_world_size()))
74 _CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
75 backend="gloo")
76 + global _LOCAL_RANK
77 + _LOCAL_RANK = local_rank
78
79
80 def initialize_model_parallel(
vllm/distributed/utils.py
+86 -1
@@ -2,9 +2,18 @@
2 # Adapted from
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 -from typing import Sequence
5 +import json
6 +import os
7 +from typing import Dict, Optional, Sequence
8
9 import torch
10 +import torch.distributed as dist
11 +
12 +from vllm.logger import init_logger
13 +
14 +from .parallel_state import get_cpu_world_group, get_local_rank
15 +
16 +logger = init_logger(__name__)
17
18
19 def ensure_divisibility(numerator, denominator):
@@ -46,3 +55,79 @@ def split_tensor_along_last_dim(
55 return tuple(chunk.contiguous() for chunk in tensor_list)
56
57 return tensor_list
58 +
59 +
60 +# code partly borrowed from
61 +# https://github.com/turboderp/exllamav2/blob/1c67f97f3d2a968605a9c31ab791a05c85bb7879/exllamav2/compat.py#L10
62 +# License: MIT
63 +def _can_actually_p2p(idx_a, idx_b):
64 + dev_i = f"cuda:{idx_a}"
65 + dev_j = f"cuda:{idx_b}"
66 + a = torch.randn(5, device=dev_i) + 123.0
67 + b = a.to(dev_j)
68 + c = b.to(dev_i)
69 + return torch.all(a == c).cpu().item()
70 +
71 +
72 +# why do we need this cache?
73 +# 1. we can have runtime checks for P2P access, where every process checks
74 +# P2P access to all other GPUs. Unfortunately, the test might cost many
75 +# (world_size * world_size) cuda context, and reduce the memory available
76 +# for the model. see https://github.com/vllm-project/vllm/issues/3821
77 +# 2. alternatively, we can have a p2p map that is generated by the master
78 +# process and broadcasted to all other processes. This still requires
79 +# #world_size of cuda context, belonging to the master process, on each GPU.
80 +# 3. we can have a cache file, that records the p2p access status. The first
81 +# time the master process checks the p2p access, it will generate the cache
82 +# file, at the cost of #world_size of cuda context. Later on, all processes
83 +# can read the cache file to check the p2p access status without any cost of
84 +# additional cuda context.
85 +# Note that the cache file is suffixed by the CUDA_VISIBLE_DEVICES, so that we
86 +# can have different cache files for different CUDA_VISIBLE_DEVICES settings,
87 +# e.g. used by different vllm engines. The device id in the cache file is a
88 +# **local** device id, i.e. from 0 to num_dev-1, where num_dev is the number
89 +# of visible devices in the vllm engine.
90 +_gpu_p2p_access_cache: Optional[Dict[str, bool]] = None
91 +
92 +
93 +def gpu_p2p_access_check(i: int, j: int) -> bool:
94 + """Check if GPU i can access GPU j."""
95 +
96 + # if the cache variable is already calculated,
97 + # read from the cache instead of checking it again
98 + global _gpu_p2p_access_cache
99 + if _gpu_p2p_access_cache is not None:
100 + return _gpu_p2p_access_cache[f"{i}->{j}"]
101 +
102 + is_distributed = dist.is_initialized()
103 +
104 + num_dev = torch.cuda.device_count()
105 + cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None)
106 + if cuda_visible_devices is None:
107 + cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
108 + path = os.path.expanduser(
109 + f"~/.config/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json")
110 + os.makedirs(os.path.dirname(path), exist_ok=True)
111 + if (not is_distributed or get_local_rank() == 0) \
112 + and (not os.path.exists(path)):
113 + # only the local master process (with local_rank == 0) can
114 + # enter this block to calculate the cache
115 + logger.info(f"generating GPU P2P access cache for in {path}")
116 + cache = {}
117 + for _i in range(num_dev):
118 + for _j in range(num_dev):
119 + # on some platforms, P2P support might be buggy and we need
120 + # additional checks. See also:
121 + # https://github.com/vllm-project/vllm/issues/2728
122 + cache[f"{_i}->{_j}"] = torch.cuda.can_device_access_peer(
123 + _i, _j) and _can_actually_p2p(_i, _j)
124 + with open(path, "w") as f:
125 + json.dump(cache, f, indent=4)
126 + if is_distributed:
127 + cpu_world_group = get_cpu_world_group()
128 + dist.barrier(cpu_world_group)
129 + logger.info(f"reading GPU P2P access cache from {path}")
130 + with open(path, "r") as f:
131 + cache = json.load(f)
132 + _gpu_p2p_access_cache = cache
133 + return _gpu_p2p_access_cache[f"{i}->{j}"]