A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[distributed] add function to create ipc buffers directly (#10064)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 5, 2024 at 22:35 UTC
4be3a45158a7fb707973d4b00410e0d2981e6825
3 files changed
+91
.buildkite/test-pipeline.yaml
+1
@@ -510,6 +510,7 @@ steps:
510
# NOTE: don't test llama model here, it seems hf implementation is buggy
511
# see https://github.com/vllm-project/vllm/pull/5689 for details
512
- pytest -v -s distributed/test_custom_all_reduce.py
513
+ - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
514
- TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m distributed_2_gpus
515
- pytest -v -s -x lora/test_mixtral.py
516
tests/distributed/test_ca_buffer_sharing.py
new
+59
@@ -0,0 +1,59 @@
1
+# can only run on machines with p2p access across GPUs
2
+# can only run with torchrun:
3
+# torchrun --nproc_per_node=2 tests/distributed/test_ca_buffer_sharing.py
4
+
5
+import ctypes
6
+
7
+import torch
8
+import torch.distributed as dist
9
+
10
+from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
11
+from vllm.distributed.device_communicators.custom_all_reduce import ( # noqa
12
+ CustomAllreduce)
13
+
14
+# create a cpu process group for communicating metadata (ipc handle)
15
+dist.init_process_group(backend="gloo")
16
+rank = local_rank = dist.get_rank()
17
+world_size = dist.get_world_size()
18
+
19
+# every process sets its own device (differently)
20
+lib = CudaRTLibrary()
21
+lib.cudaSetDevice(rank)
22
+
23
+buffer_size_in_bytes = 1024
24
+byte_value = 2 # the value we write to the buffer for verification
25
+
26
+pointers = CustomAllreduce.create_shared_buffer(buffer_size_in_bytes)
27
+
28
+print(f"Rank {rank} has pointers {pointers}")
29
+
30
+dist.barrier()
31
+torch.cuda.synchronize()
32
+
33
+if rank == 0:
34
+ # the first rank tries to write to all buffers
35
+ for p in pointers:
36
+ pointer = ctypes.c_void_p(p)
37
+ lib.cudaMemset(pointer, byte_value, buffer_size_in_bytes)
38
+
39
+dist.barrier()
40
+torch.cuda.synchronize()
41
+
42
+host_data = (ctypes.c_char * buffer_size_in_bytes)()
43
+
44
+# all ranks read from all buffers, and check if the data is correct
45
+for p in pointers:
46
+ pointer = ctypes.c_void_p(p)
47
+ lib.cudaMemcpy(host_data, pointer, buffer_size_in_bytes)
48
+ for i in range(buffer_size_in_bytes):
49
+ assert ord(host_data[i]) == byte_value, (
50
+ f"Rank {rank} failed"
51
+ f" to verify buffer {p}. Expected {byte_value}, "
52
+ f"got {ord(host_data[i])}")
53
+
54
+print(f"Rank {rank} verified all buffers")
55
+
56
+dist.barrier()
57
+torch.cuda.synchronize()
58
+
59
+CustomAllreduce.free_shared_buffer(pointers)
vllm/distributed/device_communicators/custom_all_reduce.py
+31
@@ -1,3 +1,4 @@
1
+import ctypes
2
from contextlib import contextmanager
3
from typing import Any, List, Optional, Union
4
@@ -7,6 +8,7 @@ from torch.distributed import ProcessGroup
8
9
import vllm.envs as envs
10
from vllm import _custom_ops as ops
11
+from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
12
from vllm.distributed.device_communicators.custom_all_reduce_utils import (
13
gpu_p2p_access_check)
14
from vllm.distributed.parallel_state import in_the_same_node_as
@@ -174,6 +176,35 @@ class CustomAllreduce:
176
offsets, rank, self.full_nvlink)
177
self.register_buffer(self.buffer)
178
179
+ @staticmethod
180
+ def create_shared_buffer(
181
+ size_in_bytes: int,
182
+ group: Optional[ProcessGroup] = None) -> List[int]:
183
+ lib = CudaRTLibrary()
184
+ pointer = lib.cudaMalloc(size_in_bytes)
185
+ handle = lib.cudaIpcGetMemHandle(pointer)
186
+ world_size = dist.get_world_size(group=group)
187
+ rank = dist.get_rank(group=group)
188
+ handles = [None] * world_size
189
+ dist.all_gather_object(handles, handle, group=group)
190
+
191
+ pointers: List[int] = []
192
+ for i, h in enumerate(handles):
193
+ if i == rank:
194
+ pointers.append(pointer.value) # type: ignore
195
+ else:
196
+ pointers.append(
197
+ lib.cudaIpcOpenMemHandle(h).value) # type: ignore
198
+
199
+ return pointers
200
+
201
+ @staticmethod
202
+ def free_shared_buffer(pointers: List[int],
203
+ group: Optional[ProcessGroup] = None) -> None:
204
+ rank = dist.get_rank(group=group)
205
+ lib = CudaRTLibrary()
206
+ lib.cudaFree(ctypes.c_void_p(pointers[rank]))
207
+
208
@contextmanager
209
def capture(self):
210
"""