@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4be3a45158
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[distributed] add function to create ipc buffers directly (#10064)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 5, 2024 at 22:35 UTC 4be3a45158a7fb707973d4b00410e0d2981e6825
3 files changed +91
.buildkite/test-pipeline.yaml
+1
@@ -510,6 +510,7 @@ steps:
510 # NOTE: don't test llama model here, it seems hf implementation is buggy
511 # see https://github.com/vllm-project/vllm/pull/5689 for details
512 - pytest -v -s distributed/test_custom_all_reduce.py
513 + - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
514 - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m distributed_2_gpus
515 - pytest -v -s -x lora/test_mixtral.py
516
tests/distributed/test_ca_buffer_sharing.py new
+59
@@ -0,0 +1,59 @@
1 +# can only run on machines with p2p access across GPUs
2 +# can only run with torchrun:
3 +# torchrun --nproc_per_node=2 tests/distributed/test_ca_buffer_sharing.py
4 +
5 +import ctypes
6 +
7 +import torch
8 +import torch.distributed as dist
9 +
10 +from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
11 +from vllm.distributed.device_communicators.custom_all_reduce import ( # noqa
12 + CustomAllreduce)
13 +
14 +# create a cpu process group for communicating metadata (ipc handle)
15 +dist.init_process_group(backend="gloo")
16 +rank = local_rank = dist.get_rank()
17 +world_size = dist.get_world_size()
18 +
19 +# every process sets its own device (differently)
20 +lib = CudaRTLibrary()
21 +lib.cudaSetDevice(rank)
22 +
23 +buffer_size_in_bytes = 1024
24 +byte_value = 2 # the value we write to the buffer for verification
25 +
26 +pointers = CustomAllreduce.create_shared_buffer(buffer_size_in_bytes)
27 +
28 +print(f"Rank {rank} has pointers {pointers}")
29 +
30 +dist.barrier()
31 +torch.cuda.synchronize()
32 +
33 +if rank == 0:
34 + # the first rank tries to write to all buffers
35 + for p in pointers:
36 + pointer = ctypes.c_void_p(p)
37 + lib.cudaMemset(pointer, byte_value, buffer_size_in_bytes)
38 +
39 +dist.barrier()
40 +torch.cuda.synchronize()
41 +
42 +host_data = (ctypes.c_char * buffer_size_in_bytes)()
43 +
44 +# all ranks read from all buffers, and check if the data is correct
45 +for p in pointers:
46 + pointer = ctypes.c_void_p(p)
47 + lib.cudaMemcpy(host_data, pointer, buffer_size_in_bytes)
48 + for i in range(buffer_size_in_bytes):
49 + assert ord(host_data[i]) == byte_value, (
50 + f"Rank {rank} failed"
51 + f" to verify buffer {p}. Expected {byte_value}, "
52 + f"got {ord(host_data[i])}")
53 +
54 +print(f"Rank {rank} verified all buffers")
55 +
56 +dist.barrier()
57 +torch.cuda.synchronize()
58 +
59 +CustomAllreduce.free_shared_buffer(pointers)
vllm/distributed/device_communicators/custom_all_reduce.py
+31
@@ -1,3 +1,4 @@
1 +import ctypes
2 from contextlib import contextmanager
3 from typing import Any, List, Optional, Union
4
@@ -7,6 +8,7 @@ from torch.distributed import ProcessGroup
8
9 import vllm.envs as envs
10 from vllm import _custom_ops as ops
11 +from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
12 from vllm.distributed.device_communicators.custom_all_reduce_utils import (
13 gpu_p2p_access_check)
14 from vllm.distributed.parallel_state import in_the_same_node_as
@@ -174,6 +176,35 @@ class CustomAllreduce:
176 offsets, rank, self.full_nvlink)
177 self.register_buffer(self.buffer)
178
179 + @staticmethod
180 + def create_shared_buffer(
181 + size_in_bytes: int,
182 + group: Optional[ProcessGroup] = None) -> List[int]:
183 + lib = CudaRTLibrary()
184 + pointer = lib.cudaMalloc(size_in_bytes)
185 + handle = lib.cudaIpcGetMemHandle(pointer)
186 + world_size = dist.get_world_size(group=group)
187 + rank = dist.get_rank(group=group)
188 + handles = [None] * world_size
189 + dist.all_gather_object(handles, handle, group=group)
190 +
191 + pointers: List[int] = []
192 + for i, h in enumerate(handles):
193 + if i == rank:
194 + pointers.append(pointer.value) # type: ignore
195 + else:
196 + pointers.append(
197 + lib.cudaIpcOpenMemHandle(h).value) # type: ignore
198 +
199 + return pointers
200 +
201 + @staticmethod
202 + def free_shared_buffer(pointers: List[int],
203 + group: Optional[ProcessGroup] = None) -> None:
204 + rank = dist.get_rank(group=group)
205 + lib = CudaRTLibrary()
206 + lib.cudaFree(ctypes.c_void_p(pointers[rank]))
207 +
208 @contextmanager
209 def capture(self):
210 """