@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f5bb85b435
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Core][Distributed] improve p2p cache generation (#5528)

youkaichao committed Jun 14, 2024 at 14:47 UTC f5bb85b435e6fe3db57fae1e25e09914015ef957
2 files changed +265 -96
vllm/distributed/device_communicators/cuda_wrapper.py new
+146
@@ -0,0 +1,146 @@
1 +"""This file is a pure Python wrapper for the cudart library.
2 +It avoids the need to compile a separate shared library, and is
3 +convenient for use when we just need to call a few functions.
4 +"""
5 +
6 +import ctypes
7 +from dataclasses import dataclass
8 +from typing import Any, Dict, List, Optional
9 +
10 +# this line makes it possible to directly load `libcudart.so` using `ctypes`
11 +import torch # noqa
12 +
13 +from vllm.logger import init_logger
14 +
15 +logger = init_logger(__name__)
16 +
17 +# === export types and functions from cudart to Python ===
18 +# for the original cudart definition, please check
19 +# https://docs.nvidia.com/cuda/cuda-runtime-api/index.html
20 +
21 +cudaError_t = ctypes.c_int
22 +cudaMemcpyKind = ctypes.c_int
23 +
24 +
25 +class cudaIpcMemHandle_t(ctypes.Structure):
26 + _fields_ = [("internal", ctypes.c_byte * 128)]
27 +
28 +
29 +@dataclass
30 +class Function:
31 + name: str
32 + restype: Any
33 + argtypes: List[Any]
34 +
35 +
36 +class CudaRTLibrary:
37 + exported_functions = [
38 + # ​cudaError_t cudaSetDevice ( int device )
39 + Function("cudaSetDevice", cudaError_t, [ctypes.c_int]),
40 + # cudaError_t cudaDeviceSynchronize ( void )
41 + Function("cudaDeviceSynchronize", cudaError_t, []),
42 + # ​cudaError_t cudaDeviceReset ( void )
43 + Function("cudaDeviceReset", cudaError_t, []),
44 +
45 + # const char* cudaGetErrorString ( cudaError_t error )
46 + Function("cudaGetErrorString", ctypes.c_char_p, [cudaError_t]),
47 +
48 + # ​cudaError_t cudaMalloc ( void** devPtr, size_t size )
49 + Function("cudaMalloc", cudaError_t,
50 + [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t]),
51 + # ​cudaError_t cudaFree ( void* devPtr )
52 + Function("cudaFree", cudaError_t, [ctypes.c_void_p]),
53 + # ​cudaError_t cudaMemset ( void* devPtr, int value, size_t count )
54 + Function("cudaMemset", cudaError_t,
55 + [ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t]),
56 + # ​cudaError_t cudaMemcpy ( void* dst, const void* src, size_t count, cudaMemcpyKind kind ) # noqa
57 + Function("cudaMemcpy", cudaError_t, [
58 + ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, cudaMemcpyKind
59 + ]),
60 +
61 + # cudaError_t cudaIpcGetMemHandle ( cudaIpcMemHandle_t* handle, void* devPtr ) # noqa
62 + Function("cudaIpcGetMemHandle", cudaError_t,
63 + [ctypes.POINTER(cudaIpcMemHandle_t), ctypes.c_void_p]),
64 + # ​cudaError_t cudaIpcOpenMemHandle ( void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags ) # noqa
65 + Function("cudaIpcOpenMemHandle", cudaError_t, [
66 + ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint
67 + ]),
68 + ]
69 +
70 + # class attribute to store the mapping from the path to the library
71 + # to avoid loading the same library multiple times
72 + path_to_library_cache: Dict[str, Any] = {}
73 +
74 + # class attribute to store the mapping from library path
75 + # to the corresponding dictionary
76 + path_to_dict_mapping: Dict[str, Dict[str, Any]] = {}
77 +
78 + def __init__(self, so_file: Optional[str] = None):
79 + if so_file is None:
80 + assert torch.version.cuda is not None
81 + major_version = torch.version.cuda.split(".")[0]
82 + so_file = f"libcudart.so.{major_version}"
83 + if so_file not in CudaRTLibrary.path_to_library_cache:
84 + lib = ctypes.CDLL(so_file)
85 + CudaRTLibrary.path_to_library_cache[so_file] = lib
86 + self.lib = CudaRTLibrary.path_to_library_cache[so_file]
87 +
88 + if so_file not in CudaRTLibrary.path_to_dict_mapping:
89 + _funcs = {}
90 + for func in CudaRTLibrary.exported_functions:
91 + f = getattr(self.lib, func.name)
92 + f.restype = func.restype
93 + f.argtypes = func.argtypes
94 + _funcs[func.name] = f
95 + CudaRTLibrary.path_to_dict_mapping[so_file] = _funcs
96 + self.funcs = CudaRTLibrary.path_to_dict_mapping[so_file]
97 +
98 + def CUDART_CHECK(self, result: cudaError_t) -> None:
99 + if result != 0:
100 + error_str = self.cudaGetErrorString(result)
101 + raise RuntimeError(f"CUDART error: {error_str}")
102 +
103 + def cudaGetErrorString(self, error: cudaError_t) -> str:
104 + return self.funcs["cudaGetErrorString"](error).decode("utf-8")
105 +
106 + def cudaSetDevice(self, device: int) -> None:
107 + self.CUDART_CHECK(self.funcs["cudaSetDevice"](device))
108 +
109 + def cudaDeviceSynchronize(self) -> None:
110 + self.CUDART_CHECK(self.funcs["cudaDeviceSynchronize"]())
111 +
112 + def cudaDeviceReset(self) -> None:
113 + self.CUDART_CHECK(self.funcs["cudaDeviceReset"]())
114 +
115 + def cudaMalloc(self, size: int) -> ctypes.c_void_p:
116 + devPtr = ctypes.c_void_p()
117 + self.CUDART_CHECK(self.funcs["cudaMalloc"](ctypes.byref(devPtr), size))
118 + return devPtr
119 +
120 + def cudaFree(self, devPtr: ctypes.c_void_p) -> None:
121 + self.CUDART_CHECK(self.funcs["cudaFree"](devPtr))
122 +
123 + def cudaMemset(self, devPtr: ctypes.c_void_p, value: int,
124 + count: int) -> None:
125 + self.CUDART_CHECK(self.funcs["cudaMemset"](devPtr, value, count))
126 +
127 + def cudaMemcpy(self, dst: ctypes.c_void_p, src: ctypes.c_void_p,
128 + count: int) -> None:
129 + cudaMemcpyDefault = 4
130 + kind = cudaMemcpyDefault
131 + self.CUDART_CHECK(self.funcs["cudaMemcpy"](dst, src, count, kind))
132 +
133 + def cudaIpcGetMemHandle(self,
134 + devPtr: ctypes.c_void_p) -> cudaIpcMemHandle_t:
135 + handle = cudaIpcMemHandle_t()
136 + self.CUDART_CHECK(self.funcs["cudaIpcGetMemHandle"](
137 + ctypes.byref(handle), devPtr))
138 + return handle
139 +
140 + def cudaIpcOpenMemHandle(self,
141 + handle: cudaIpcMemHandle_t) -> ctypes.c_void_p:
142 + cudaIpcMemLazyEnablePeerAccess = 1
143 + devPtr = ctypes.c_void_p()
144 + self.CUDART_CHECK(self.funcs["cudaIpcOpenMemHandle"](
145 + ctypes.byref(devPtr), handle, cudaIpcMemLazyEnablePeerAccess))
146 + return devPtr
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+119 -96
@@ -1,87 +1,98 @@
1 +import ctypes
2 import json
3 import os
3 -import sys
4 -import tempfile
5 -import time
6 -from contextlib import contextmanager
7 -from typing import Callable, Dict, List, Optional
4 +from itertools import product
5 +from typing import Dict, Optional, Sequence
6
9 -import torch
7 import torch.distributed as dist
8 import torch.multiprocessing as mp
9
10 import vllm.envs as envs
11 +from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
12 from vllm.logger import init_logger
13 from vllm.utils import cuda_device_count_stateless
14
15 logger = init_logger(__name__)
16
17
20 -@contextmanager
21 -def mute_output():
22 - with open(os.devnull, "w") as f:
23 - sys.stderr = f
24 - sys.stdout = f
25 - yield
26 -
27 -
28 -def producer(i: int,
29 - init_method: str,
18 +def producer(batch_src: Sequence[int],
19 + producer_queue,
20 + consumer_queue,
21 + result_queue,
22 cuda_visible_devices: Optional[str] = None):
23 if cuda_visible_devices is not None:
24 os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
33 - with mute_output():
34 - dist.init_process_group(
35 - backend="gloo",
36 - init_method=init_method,
37 - world_size=2,
38 - rank=0,
39 - )
40 - # produce a tensor in GPU i
41 - data = torch.zeros((128, ), device=f"cuda:{i}")
42 - # get the information to reconstruct the shared tensor
43 - func, args = torch.multiprocessing.reductions.reduce_tensor(data)
44 - args = list(args)
45 - dist.broadcast_object_list([(func, args)], src=0)
46 - dist.barrier()
47 - torch.cuda.synchronize()
48 - assert torch.all(data == 1).item()
49 -
50 -
51 -def consumer(j: int,
52 - init_method: str,
25 +
26 + lib = CudaRTLibrary()
27 + for i in batch_src:
28 + lib.cudaSetDevice(i)
29 + pointer = lib.cudaMalloc(1024)
30 + lib.cudaMemset(pointer, 1, 1024)
31 + lib.cudaDeviceSynchronize()
32 + handle = lib.cudaIpcGetMemHandle(pointer)
33 + producer_queue.put(handle)
34 + open_success = consumer_queue.get()
35 + if open_success:
36 + # use two queues to simulate barrier
37 + producer_queue.put(0)
38 + consumer_queue.get()
39 + # check if the memory is modified
40 + host_data = (ctypes.c_char * 1024)()
41 + lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore
42 + for i in range(1024):
43 + if ord(host_data[i]) != 2:
44 + open_success = False
45 + break
46 + result_queue.put(open_success)
47 + lib.cudaDeviceReset()
48 +
49 +
50 +def consumer(batch_tgt: Sequence[int],
51 + producer_queue,
52 + consumer_queue,
53 + result_queue,
54 cuda_visible_devices: Optional[str] = None):
55 if cuda_visible_devices is not None:
56 os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
56 - with mute_output():
57 - dist.init_process_group(
58 - backend="gloo",
59 - init_method=init_method,
60 - world_size=2,
61 - rank=1,
62 - )
63 - torch.cuda.set_device(j)
64 - recv = [None]
65 - dist.broadcast_object_list(recv, src=0)
66 - func: Callable
67 - args: List
68 - func, args = recv[0] # type: ignore
69 - # `args[6]` is the device id
70 - # by default pytorch will use `i` from the producer
71 - # here we need to set it to `j` to test P2P access
72 - args[6] = j
73 - data = func(*args)
74 - data += 1
75 - dist.barrier()
76 - torch.cuda.synchronize()
77 - assert torch.all(data == 1).item()
78 -
79 -
80 -def can_actually_p2p(i, j):
57 +
58 + lib = CudaRTLibrary()
59 + for j in batch_tgt:
60 + lib.cudaSetDevice(j)
61 + handle = producer_queue.get()
62 + open_success = False
63 + try:
64 + pointer = lib.cudaIpcOpenMemHandle(handle) # type: ignore
65 + open_success = True
66 + except RuntimeError:
67 + # cannot error out here, because the producer process
68 + # is still waiting for the response.
69 + pass
70 + consumer_queue.put(open_success)
71 + if open_success:
72 + # modify the memory
73 + lib.cudaMemset(pointer, 2, 1024)
74 + # use two queues to simulate barrier
75 + producer_queue.get()
76 + consumer_queue.put(0)
77 + # check if the memory is modified
78 + host_data = (ctypes.c_char * 1024)()
79 + lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore
80 + for i in range(1024):
81 + if ord(host_data[i]) != 2:
82 + open_success = False
83 + break
84 + result_queue.put(open_success)
85 + lib.cudaDeviceReset()
86 +
87 +
88 +def can_actually_p2p(
89 + batch_src: Sequence[int],
90 + batch_tgt: Sequence[int],
91 +):
92 """
93 Usually, checking if P2P access is enabled can be done by
83 - `torch.cuda.can_device_access_peer(i, j)`. However, sometimes
84 - the driver might be broken, and `torch.cuda.can_device_access_peer(i, j)`
94 + `torch.cuda.can_device_access_peer(src, tgt)`. However, sometimes
95 + the driver might be broken, and `torch.cuda.can_device_access_peer(src, tgt)`
96 returns `True` even if P2P access is not actually possible.
97 See https://github.com/vllm-project/vllm/issues/2728 and
98 https://forums.developer.nvidia.com/t/direct-gpu-gpu-communication-does-not-seem-to-work-properly/283264/10
@@ -90,41 +101,50 @@ def can_actually_p2p(i, j):
101
102 Note on p2p and cuda IPC:
103 Usually, one process uses one GPU:
93 - GPU i --> cuda context i --> tensor i --> process i
104 + GPU src --> cuda context src --> tensor src --> process src
105
106 We need to combine p2p and cuda IPC, so that:
96 - GPU i --> cuda context i --> tensor i --> process i
97 - |shared|
98 - GPU j --> cuda context j --> tensor j --> process j
99 - That is to say, process i creates a tensor in GPU i, passes IPC handle to
100 - process j, and process j accesses the tensor in GPU j. Any operation on the
101 - tensor in process j will be reflected in the tensor in process i, because
107 + GPU src --> cuda context src --> tensor src --> process src
108 + |shared|
109 + GPU tgt --> cuda context tgt --> tensor tgt --> process tgt
110 + That is to say, process src creates a tensor in GPU src, passes IPC handle to
111 + process tgt, and process tgt accesses the tensor in GPU tgt. Any operation on the
112 + tensor in process tgt will be reflected in the tensor in process src, because
113 they are the same memory segment.
103 - It is important to note that process j accesses the tensor in GPU j, not
104 - GPU i. That's why we need p2p access. # noqa
105 - """
114 + It is important to note that process tgt accesses the tensor in GPU tgt, not
115 + GPU src. That's why we need p2p access.
116 +
117 + The most time-consuming part is the process creation. To avoid creating
118 + processes for every pair of GPUs, we use batched testing. We create two
119 + processes for testing all pairs of GPUs in batch. The trick is to reset
120 + the device after each test (which is not available in PyTorch).
121 + """ # noqa
122 cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', None)
123 # pass the CUDA_VISIBLE_DEVICES to the child process
124 # to make sure they see the same set of GPUs
125
110 - # make sure the temp file is not the same across different calls
111 - temp_path = tempfile.mktemp() + str(time.time())
112 - # create an empty file
113 - with open(temp_path, "w"):
114 - pass
115 - init_method = f"file://{temp_path}"
116 -
126 # make sure the processes are spawned
127 smp = mp.get_context("spawn")
119 - pi = smp.Process(target=producer,
120 - args=(i, init_method, cuda_visible_devices))
121 - pj = smp.Process(target=consumer,
122 - args=(j, init_method, cuda_visible_devices))
123 - pi.start()
124 - pj.start()
125 - pi.join()
126 - pj.join()
127 - return pi.exitcode == 0 and pj.exitcode == 0
128 + producer_queue = smp.Queue()
129 + consumer_queue = smp.Queue()
130 + result_queue = smp.Queue()
131 + p_src = smp.Process(target=producer,
132 + args=(batch_src, producer_queue, consumer_queue,
133 + result_queue, cuda_visible_devices))
134 + p_tgt = smp.Process(target=consumer,
135 + args=(batch_tgt, producer_queue, consumer_queue,
136 + result_queue, cuda_visible_devices))
137 + p_src.start()
138 + p_tgt.start()
139 + p_src.join()
140 + p_tgt.join()
141 + result = []
142 + for src, tgt in zip(batch_src, batch_tgt):
143 + a = result_queue.get()
144 + b = result_queue.get()
145 + assert a == b
146 + result.append(a)
147 + return result
148
149
150 # why do we need this cache?
@@ -142,14 +162,14 @@ def can_actually_p2p(i, j):
162 _gpu_p2p_access_cache: Optional[Dict[str, bool]] = None
163
164
145 -def gpu_p2p_access_check(i: int, j: int) -> bool:
146 - """Check if GPU i can access GPU j."""
165 +def gpu_p2p_access_check(src: int, tgt: int) -> bool:
166 + """Check if GPU src can access GPU tgt."""
167
168 # if the cache variable is already calculated,
169 # read from the cache instead of checking it again
170 global _gpu_p2p_access_cache
171 if _gpu_p2p_access_cache is not None:
152 - return _gpu_p2p_access_cache[f"{i}->{j}"]
172 + return _gpu_p2p_access_cache[f"{src}->{tgt}"]
173
174 is_distributed = dist.is_initialized()
175
@@ -169,9 +189,12 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
189 # enter this block to calculate the cache
190 logger.info("generating GPU P2P access cache in %s", path)
191 cache = {}
172 - for _i in range(num_dev):
173 - for _j in range(num_dev):
174 - cache[f"{_i}->{_j}"] = can_actually_p2p(_i, _j)
192 + ids = list(range(num_dev))
193 + # batch of all pairs of GPUs
194 + batch_src, batch_tgt = zip(*list(product(ids, ids)))
195 + result = can_actually_p2p(batch_src, batch_tgt)
196 + for _i, _j, r in zip(batch_src, batch_tgt, result):
197 + cache[f"{_i}->{_j}"] = r
198 with open(path, "w") as f:
199 json.dump(cache, f, indent=4)
200 if is_distributed:
@@ -180,7 +203,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
203 with open(path, "r") as f:
204 cache = json.load(f)
205 _gpu_p2p_access_cache = cache
183 - return _gpu_p2p_access_cache[f"{i}->{j}"]
206 + return _gpu_p2p_access_cache[f"{src}->{tgt}"]
207
208
209 __all__ = ["gpu_p2p_access_check"]