A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core][Distributed] improve p2p cache generation (#5528)
youkaichao committed
Jun 14, 2024 at 14:47 UTC
f5bb85b435e6fe3db57fae1e25e09914015ef957
2 files changed
+265
-96
vllm/distributed/device_communicators/cuda_wrapper.py
new
+146
@@ -0,0 +1,146 @@
1
+"""This file is a pure Python wrapper for the cudart library.
2
+It avoids the need to compile a separate shared library, and is
3
+convenient for use when we just need to call a few functions.
4
+"""
5
+
6
+import ctypes
7
+from dataclasses import dataclass
8
+from typing import Any, Dict, List, Optional
9
+
10
+# this line makes it possible to directly load `libcudart.so` using `ctypes`
11
+import torch # noqa
12
+
13
+from vllm.logger import init_logger
14
+
15
+logger = init_logger(__name__)
16
+
17
+# === export types and functions from cudart to Python ===
18
+# for the original cudart definition, please check
19
+# https://docs.nvidia.com/cuda/cuda-runtime-api/index.html
20
+
21
+cudaError_t = ctypes.c_int
22
+cudaMemcpyKind = ctypes.c_int
23
+
24
+
25
+class cudaIpcMemHandle_t(ctypes.Structure):
26
+ _fields_ = [("internal", ctypes.c_byte * 128)]
27
+
28
+
29
+@dataclass
30
+class Function:
31
+ name: str
32
+ restype: Any
33
+ argtypes: List[Any]
34
+
35
+
36
+class CudaRTLibrary:
37
+ exported_functions = [
38
+ # cudaError_t cudaSetDevice ( int device )
39
+ Function("cudaSetDevice", cudaError_t, [ctypes.c_int]),
40
+ # cudaError_t cudaDeviceSynchronize ( void )
41
+ Function("cudaDeviceSynchronize", cudaError_t, []),
42
+ # cudaError_t cudaDeviceReset ( void )
43
+ Function("cudaDeviceReset", cudaError_t, []),
44
+
45
+ # const char* cudaGetErrorString ( cudaError_t error )
46
+ Function("cudaGetErrorString", ctypes.c_char_p, [cudaError_t]),
47
+
48
+ # cudaError_t cudaMalloc ( void** devPtr, size_t size )
49
+ Function("cudaMalloc", cudaError_t,
50
+ [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t]),
51
+ # cudaError_t cudaFree ( void* devPtr )
52
+ Function("cudaFree", cudaError_t, [ctypes.c_void_p]),
53
+ # cudaError_t cudaMemset ( void* devPtr, int value, size_t count )
54
+ Function("cudaMemset", cudaError_t,
55
+ [ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t]),
56
+ # cudaError_t cudaMemcpy ( void* dst, const void* src, size_t count, cudaMemcpyKind kind ) # noqa
57
+ Function("cudaMemcpy", cudaError_t, [
58
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, cudaMemcpyKind
59
+ ]),
60
+
61
+ # cudaError_t cudaIpcGetMemHandle ( cudaIpcMemHandle_t* handle, void* devPtr ) # noqa
62
+ Function("cudaIpcGetMemHandle", cudaError_t,
63
+ [ctypes.POINTER(cudaIpcMemHandle_t), ctypes.c_void_p]),
64
+ # cudaError_t cudaIpcOpenMemHandle ( void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags ) # noqa
65
+ Function("cudaIpcOpenMemHandle", cudaError_t, [
66
+ ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint
67
+ ]),
68
+ ]
69
+
70
+ # class attribute to store the mapping from the path to the library
71
+ # to avoid loading the same library multiple times
72
+ path_to_library_cache: Dict[str, Any] = {}
73
+
74
+ # class attribute to store the mapping from library path
75
+ # to the corresponding dictionary
76
+ path_to_dict_mapping: Dict[str, Dict[str, Any]] = {}
77
+
78
+ def __init__(self, so_file: Optional[str] = None):
79
+ if so_file is None:
80
+ assert torch.version.cuda is not None
81
+ major_version = torch.version.cuda.split(".")[0]
82
+ so_file = f"libcudart.so.{major_version}"
83
+ if so_file not in CudaRTLibrary.path_to_library_cache:
84
+ lib = ctypes.CDLL(so_file)
85
+ CudaRTLibrary.path_to_library_cache[so_file] = lib
86
+ self.lib = CudaRTLibrary.path_to_library_cache[so_file]
87
+
88
+ if so_file not in CudaRTLibrary.path_to_dict_mapping:
89
+ _funcs = {}
90
+ for func in CudaRTLibrary.exported_functions:
91
+ f = getattr(self.lib, func.name)
92
+ f.restype = func.restype
93
+ f.argtypes = func.argtypes
94
+ _funcs[func.name] = f
95
+ CudaRTLibrary.path_to_dict_mapping[so_file] = _funcs
96
+ self.funcs = CudaRTLibrary.path_to_dict_mapping[so_file]
97
+
98
+ def CUDART_CHECK(self, result: cudaError_t) -> None:
99
+ if result != 0:
100
+ error_str = self.cudaGetErrorString(result)
101
+ raise RuntimeError(f"CUDART error: {error_str}")
102
+
103
+ def cudaGetErrorString(self, error: cudaError_t) -> str:
104
+ return self.funcs["cudaGetErrorString"](error).decode("utf-8")
105
+
106
+ def cudaSetDevice(self, device: int) -> None:
107
+ self.CUDART_CHECK(self.funcs["cudaSetDevice"](device))
108
+
109
+ def cudaDeviceSynchronize(self) -> None:
110
+ self.CUDART_CHECK(self.funcs["cudaDeviceSynchronize"]())
111
+
112
+ def cudaDeviceReset(self) -> None:
113
+ self.CUDART_CHECK(self.funcs["cudaDeviceReset"]())
114
+
115
+ def cudaMalloc(self, size: int) -> ctypes.c_void_p:
116
+ devPtr = ctypes.c_void_p()
117
+ self.CUDART_CHECK(self.funcs["cudaMalloc"](ctypes.byref(devPtr), size))
118
+ return devPtr
119
+
120
+ def cudaFree(self, devPtr: ctypes.c_void_p) -> None:
121
+ self.CUDART_CHECK(self.funcs["cudaFree"](devPtr))
122
+
123
+ def cudaMemset(self, devPtr: ctypes.c_void_p, value: int,
124
+ count: int) -> None:
125
+ self.CUDART_CHECK(self.funcs["cudaMemset"](devPtr, value, count))
126
+
127
+ def cudaMemcpy(self, dst: ctypes.c_void_p, src: ctypes.c_void_p,
128
+ count: int) -> None:
129
+ cudaMemcpyDefault = 4
130
+ kind = cudaMemcpyDefault
131
+ self.CUDART_CHECK(self.funcs["cudaMemcpy"](dst, src, count, kind))
132
+
133
+ def cudaIpcGetMemHandle(self,
134
+ devPtr: ctypes.c_void_p) -> cudaIpcMemHandle_t:
135
+ handle = cudaIpcMemHandle_t()
136
+ self.CUDART_CHECK(self.funcs["cudaIpcGetMemHandle"](
137
+ ctypes.byref(handle), devPtr))
138
+ return handle
139
+
140
+ def cudaIpcOpenMemHandle(self,
141
+ handle: cudaIpcMemHandle_t) -> ctypes.c_void_p:
142
+ cudaIpcMemLazyEnablePeerAccess = 1
143
+ devPtr = ctypes.c_void_p()
144
+ self.CUDART_CHECK(self.funcs["cudaIpcOpenMemHandle"](
145
+ ctypes.byref(devPtr), handle, cudaIpcMemLazyEnablePeerAccess))
146
+ return devPtr
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+119
-96
@@ -1,87 +1,98 @@
1
+import ctypes
2
import json
3
import os
3
-import sys
4
-import tempfile
5
-import time
6
-from contextlib import contextmanager
7
-from typing import Callable, Dict, List, Optional
4
+from itertools import product
5
+from typing import Dict, Optional, Sequence
6
9
-import torch
7
import torch.distributed as dist
8
import torch.multiprocessing as mp
9
10
import vllm.envs as envs
11
+from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
12
from vllm.logger import init_logger
13
from vllm.utils import cuda_device_count_stateless
14
15
logger = init_logger(__name__)
16
17
20
-@contextmanager
21
-def mute_output():
22
- with open(os.devnull, "w") as f:
23
- sys.stderr = f
24
- sys.stdout = f
25
- yield
26
-
27
-
28
-def producer(i: int,
29
- init_method: str,
18
+def producer(batch_src: Sequence[int],
19
+ producer_queue,
20
+ consumer_queue,
21
+ result_queue,
22
cuda_visible_devices: Optional[str] = None):
23
if cuda_visible_devices is not None:
24
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
33
- with mute_output():
34
- dist.init_process_group(
35
- backend="gloo",
36
- init_method=init_method,
37
- world_size=2,
38
- rank=0,
39
- )
40
- # produce a tensor in GPU i
41
- data = torch.zeros((128, ), device=f"cuda:{i}")
42
- # get the information to reconstruct the shared tensor
43
- func, args = torch.multiprocessing.reductions.reduce_tensor(data)
44
- args = list(args)
45
- dist.broadcast_object_list([(func, args)], src=0)
46
- dist.barrier()
47
- torch.cuda.synchronize()
48
- assert torch.all(data == 1).item()
49
-
50
-
51
-def consumer(j: int,
52
- init_method: str,
25
+
26
+ lib = CudaRTLibrary()
27
+ for i in batch_src:
28
+ lib.cudaSetDevice(i)
29
+ pointer = lib.cudaMalloc(1024)
30
+ lib.cudaMemset(pointer, 1, 1024)
31
+ lib.cudaDeviceSynchronize()
32
+ handle = lib.cudaIpcGetMemHandle(pointer)
33
+ producer_queue.put(handle)
34
+ open_success = consumer_queue.get()
35
+ if open_success:
36
+ # use two queues to simulate barrier
37
+ producer_queue.put(0)
38
+ consumer_queue.get()
39
+ # check if the memory is modified
40
+ host_data = (ctypes.c_char * 1024)()
41
+ lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore
42
+ for i in range(1024):
43
+ if ord(host_data[i]) != 2:
44
+ open_success = False
45
+ break
46
+ result_queue.put(open_success)
47
+ lib.cudaDeviceReset()
48
+
49
+
50
+def consumer(batch_tgt: Sequence[int],
51
+ producer_queue,
52
+ consumer_queue,
53
+ result_queue,
54
cuda_visible_devices: Optional[str] = None):
55
if cuda_visible_devices is not None:
56
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
56
- with mute_output():
57
- dist.init_process_group(
58
- backend="gloo",
59
- init_method=init_method,
60
- world_size=2,
61
- rank=1,
62
- )
63
- torch.cuda.set_device(j)
64
- recv = [None]
65
- dist.broadcast_object_list(recv, src=0)
66
- func: Callable
67
- args: List
68
- func, args = recv[0] # type: ignore
69
- # `args[6]` is the device id
70
- # by default pytorch will use `i` from the producer
71
- # here we need to set it to `j` to test P2P access
72
- args[6] = j
73
- data = func(*args)
74
- data += 1
75
- dist.barrier()
76
- torch.cuda.synchronize()
77
- assert torch.all(data == 1).item()
78
-
79
-
80
-def can_actually_p2p(i, j):
57
+
58
+ lib = CudaRTLibrary()
59
+ for j in batch_tgt:
60
+ lib.cudaSetDevice(j)
61
+ handle = producer_queue.get()
62
+ open_success = False
63
+ try:
64
+ pointer = lib.cudaIpcOpenMemHandle(handle) # type: ignore
65
+ open_success = True
66
+ except RuntimeError:
67
+ # cannot error out here, because the producer process
68
+ # is still waiting for the response.
69
+ pass
70
+ consumer_queue.put(open_success)
71
+ if open_success:
72
+ # modify the memory
73
+ lib.cudaMemset(pointer, 2, 1024)
74
+ # use two queues to simulate barrier
75
+ producer_queue.get()
76
+ consumer_queue.put(0)
77
+ # check if the memory is modified
78
+ host_data = (ctypes.c_char * 1024)()
79
+ lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore
80
+ for i in range(1024):
81
+ if ord(host_data[i]) != 2:
82
+ open_success = False
83
+ break
84
+ result_queue.put(open_success)
85
+ lib.cudaDeviceReset()
86
+
87
+
88
+def can_actually_p2p(
89
+ batch_src: Sequence[int],
90
+ batch_tgt: Sequence[int],
91
+):
92
"""
93
Usually, checking if P2P access is enabled can be done by
83
- `torch.cuda.can_device_access_peer(i, j)`. However, sometimes
84
- the driver might be broken, and `torch.cuda.can_device_access_peer(i, j)`
94
+ `torch.cuda.can_device_access_peer(src, tgt)`. However, sometimes
95
+ the driver might be broken, and `torch.cuda.can_device_access_peer(src, tgt)`
96
returns `True` even if P2P access is not actually possible.
97
See https://github.com/vllm-project/vllm/issues/2728 and
98
https://forums.developer.nvidia.com/t/direct-gpu-gpu-communication-does-not-seem-to-work-properly/283264/10
@@ -90,41 +101,50 @@ def can_actually_p2p(i, j):
101
102
Note on p2p and cuda IPC:
103
Usually, one process uses one GPU:
93
- GPU i --> cuda context i --> tensor i --> process i
104
+ GPU src --> cuda context src --> tensor src --> process src
105
106
We need to combine p2p and cuda IPC, so that:
96
- GPU i --> cuda context i --> tensor i --> process i
97
- |shared|
98
- GPU j --> cuda context j --> tensor j --> process j
99
- That is to say, process i creates a tensor in GPU i, passes IPC handle to
100
- process j, and process j accesses the tensor in GPU j. Any operation on the
101
- tensor in process j will be reflected in the tensor in process i, because
107
+ GPU src --> cuda context src --> tensor src --> process src
108
+ |shared|
109
+ GPU tgt --> cuda context tgt --> tensor tgt --> process tgt
110
+ That is to say, process src creates a tensor in GPU src, passes IPC handle to
111
+ process tgt, and process tgt accesses the tensor in GPU tgt. Any operation on the
112
+ tensor in process tgt will be reflected in the tensor in process src, because
113
they are the same memory segment.
103
- It is important to note that process j accesses the tensor in GPU j, not
104
- GPU i. That's why we need p2p access. # noqa
105
- """
114
+ It is important to note that process tgt accesses the tensor in GPU tgt, not
115
+ GPU src. That's why we need p2p access.
116
+
117
+ The most time-consuming part is the process creation. To avoid creating
118
+ processes for every pair of GPUs, we use batched testing. We create two
119
+ processes for testing all pairs of GPUs in batch. The trick is to reset
120
+ the device after each test (which is not available in PyTorch).
121
+ """ # noqa
122
cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', None)
123
# pass the CUDA_VISIBLE_DEVICES to the child process
124
# to make sure they see the same set of GPUs
125
110
- # make sure the temp file is not the same across different calls
111
- temp_path = tempfile.mktemp() + str(time.time())
112
- # create an empty file
113
- with open(temp_path, "w"):
114
- pass
115
- init_method = f"file://{temp_path}"
116
-
126
# make sure the processes are spawned
127
smp = mp.get_context("spawn")
119
- pi = smp.Process(target=producer,
120
- args=(i, init_method, cuda_visible_devices))
121
- pj = smp.Process(target=consumer,
122
- args=(j, init_method, cuda_visible_devices))
123
- pi.start()
124
- pj.start()
125
- pi.join()
126
- pj.join()
127
- return pi.exitcode == 0 and pj.exitcode == 0
128
+ producer_queue = smp.Queue()
129
+ consumer_queue = smp.Queue()
130
+ result_queue = smp.Queue()
131
+ p_src = smp.Process(target=producer,
132
+ args=(batch_src, producer_queue, consumer_queue,
133
+ result_queue, cuda_visible_devices))
134
+ p_tgt = smp.Process(target=consumer,
135
+ args=(batch_tgt, producer_queue, consumer_queue,
136
+ result_queue, cuda_visible_devices))
137
+ p_src.start()
138
+ p_tgt.start()
139
+ p_src.join()
140
+ p_tgt.join()
141
+ result = []
142
+ for src, tgt in zip(batch_src, batch_tgt):
143
+ a = result_queue.get()
144
+ b = result_queue.get()
145
+ assert a == b
146
+ result.append(a)
147
+ return result
148
149
150
# why do we need this cache?
@@ -142,14 +162,14 @@ def can_actually_p2p(i, j):
162
_gpu_p2p_access_cache: Optional[Dict[str, bool]] = None
163
164
145
-def gpu_p2p_access_check(i: int, j: int) -> bool:
146
- """Check if GPU i can access GPU j."""
165
+def gpu_p2p_access_check(src: int, tgt: int) -> bool:
166
+ """Check if GPU src can access GPU tgt."""
167
168
# if the cache variable is already calculated,
169
# read from the cache instead of checking it again
170
global _gpu_p2p_access_cache
171
if _gpu_p2p_access_cache is not None:
152
- return _gpu_p2p_access_cache[f"{i}->{j}"]
172
+ return _gpu_p2p_access_cache[f"{src}->{tgt}"]
173
174
is_distributed = dist.is_initialized()
175
@@ -169,9 +189,12 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
189
# enter this block to calculate the cache
190
logger.info("generating GPU P2P access cache in %s", path)
191
cache = {}
172
- for _i in range(num_dev):
173
- for _j in range(num_dev):
174
- cache[f"{_i}->{_j}"] = can_actually_p2p(_i, _j)
192
+ ids = list(range(num_dev))
193
+ # batch of all pairs of GPUs
194
+ batch_src, batch_tgt = zip(*list(product(ids, ids)))
195
+ result = can_actually_p2p(batch_src, batch_tgt)
196
+ for _i, _j, r in zip(batch_src, batch_tgt, result):
197
+ cache[f"{_i}->{_j}"] = r
198
with open(path, "w") as f:
199
json.dump(cache, f, indent=4)
200
if is_distributed:
@@ -180,7 +203,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
203
with open(path, "r") as f:
204
cache = json.load(f)
205
_gpu_p2p_access_cache = cache
183
- return _gpu_p2p_access_cache[f"{i}->{j}"]
206
+ return _gpu_p2p_access_cache[f"{src}->{tgt}"]
207
208
209
__all__ = ["gpu_p2p_access_check"]