@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6192e9b8fe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Core][Distributed] Refactor ipc buffer init in CustomAllreduce (#10030)

Signed-off-by: Hanzhi Zhou <hanzhi713@gmail.com>

Hanzhi Zhou committed Nov 6, 2024 at 23:50 UTC 6192e9b8fef8492c3e52bd65c7d954a1ef9b40c8
9 files changed +218 -260
csrc/custom_all_reduce.cu
+61 -58
@@ -5,32 +5,29 @@
5
6 #include "custom_all_reduce.cuh"
7
8 -// fake pointer type, must match fptr_t type in ops.h
8 +// Fake pointer type, must match fptr_t type in ops.h.
9 +// We use this type alias to indicate when pointers are passed in as int64_t.
10 using fptr_t = int64_t;
11 static_assert(sizeof(void*) == sizeof(fptr_t));
12
12 -fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
13 - const std::vector<std::string>& handles,
14 - const std::vector<int64_t>& offsets, int64_t rank,
13 +fptr_t init_custom_ar(const std::vector<fptr_t>& fake_ipc_ptrs,
14 + torch::Tensor& rank_data, int64_t rank,
15 bool full_nvlink) {
16 - int world_size = offsets.size();
16 + int world_size = fake_ipc_ptrs.size();
17 if (world_size > 8)
18 throw std::invalid_argument("world size > 8 is not supported");
19 if (world_size % 2 != 0)
20 throw std::invalid_argument("Odd num gpus is not supported for now");
21 - if (world_size != handles.size())
22 - throw std::invalid_argument(
23 - "handles length should equal to offsets length");
21 if (rank < 0 || rank >= world_size)
22 throw std::invalid_argument("invalid rank passed in");
23
27 - cudaIpcMemHandle_t ipc_handles[8];
24 + vllm::Signal* ipc_ptrs[8];
25 for (int i = 0; i < world_size; i++) {
29 - std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
26 + ipc_ptrs[i] = reinterpret_cast<vllm::Signal*>(fake_ipc_ptrs[i]);
27 }
31 - return (fptr_t) new vllm::CustomAllreduce(
32 - reinterpret_cast<vllm::Signal*>(meta.data_ptr()), rank_data.data_ptr(),
33 - rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
28 + return (fptr_t) new vllm::CustomAllreduce(ipc_ptrs, rank_data.data_ptr(),
29 + rank_data.numel(), rank, world_size,
30 + full_nvlink);
31 }
32
33 /**
@@ -55,26 +52,48 @@ bool _is_weak_contiguous(torch::Tensor& t) {
52 t.numel() * t.element_size());
53 }
54
58 -void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
59 - cudaStream_t stream) {
55 +/**
56 + * Performs an out-of-place allreduce and stores result in out.
57 + *
58 + * If _reg_buffer is null, assumes inp.data_ptr() is already IPC-registered.
59 + * Otherwise, _reg_buffer is assumed to be IPC-registered and inp is first
60 + * copied into _reg_buffer.
61 + */
62 +void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
63 + fptr_t _reg_buffer, int64_t reg_buffer_sz_bytes) {
64 auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
65 + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
66 + auto stream = c10::cuda::getCurrentCUDAStream().stream();
67 +
68 + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
69 + TORCH_CHECK_EQ(inp.numel(), out.numel());
70 TORCH_CHECK(_is_weak_contiguous(out));
71 + TORCH_CHECK(_is_weak_contiguous(inp));
72 + auto input_size = inp.numel() * inp.element_size();
73 + auto reg_buffer = reinterpret_cast<void*>(_reg_buffer);
74 + if (reg_buffer) {
75 + TORCH_CHECK_LE(input_size, reg_buffer_sz_bytes);
76 + AT_CUDA_CHECK(cudaMemcpyAsync(reg_buffer, inp.data_ptr(), input_size,
77 + cudaMemcpyDeviceToDevice, stream));
78 + } else {
79 + reg_buffer = inp.data_ptr();
80 + }
81 switch (out.scalar_type()) {
82 case at::ScalarType::Float: {
64 - fa->allreduce<float>(stream, reinterpret_cast<float*>(inp.data_ptr()),
83 + fa->allreduce<float>(stream, reinterpret_cast<float*>(reg_buffer),
84 reinterpret_cast<float*>(out.data_ptr()),
85 out.numel());
86 break;
87 }
88 case at::ScalarType::Half: {
70 - fa->allreduce<half>(stream, reinterpret_cast<half*>(inp.data_ptr()),
89 + fa->allreduce<half>(stream, reinterpret_cast<half*>(reg_buffer),
90 reinterpret_cast<half*>(out.data_ptr()), out.numel());
91 break;
92 }
93 #if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
94 case at::ScalarType::BFloat16: {
95 fa->allreduce<nv_bfloat16>(
77 - stream, reinterpret_cast<nv_bfloat16*>(inp.data_ptr()),
96 + stream, reinterpret_cast<nv_bfloat16*>(reg_buffer),
97 reinterpret_cast<nv_bfloat16*>(out.data_ptr()), out.numel());
98 break;
99 }
@@ -85,57 +104,41 @@ void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
104 }
105 }
106
88 -void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out) {
89 - const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
90 - auto stream = c10::cuda::getCurrentCUDAStream().stream();
91 - TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
92 - TORCH_CHECK_EQ(inp.numel(), out.numel());
93 - _all_reduce(_fa, inp, out, stream);
94 -}
95 -
96 -void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
97 - torch::Tensor& out) {
98 - const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
99 - auto stream = c10::cuda::getCurrentCUDAStream().stream();
100 -
101 - auto input_size = inp.numel() * inp.element_size();
102 - TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
103 - TORCH_CHECK_EQ(inp.numel(), out.numel());
104 - TORCH_CHECK(input_size <= reg_buffer.numel() * reg_buffer.element_size(),
105 - "registered buffer is too small to contain the input");
106 - AT_CUDA_CHECK(cudaMemcpyAsync(reg_buffer.data_ptr(), inp.data_ptr(),
107 - input_size, cudaMemcpyDeviceToDevice, stream));
108 - _all_reduce(_fa, reg_buffer, out, stream);
109 -}
110 -
107 void dispose(fptr_t _fa) {
112 - auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
113 - delete fa;
108 + delete reinterpret_cast<vllm::CustomAllreduce*>(_fa);
109 }
110
111 int64_t meta_size() { return sizeof(vllm::Signal); }
112
118 -void register_buffer(fptr_t _fa, torch::Tensor& t,
119 - const std::vector<std::string>& handles,
120 - const std::vector<int64_t>& offsets) {
113 +void register_buffer(fptr_t _fa, const std::vector<fptr_t>& fake_ipc_ptrs) {
114 auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
122 - fa->register_buffer(handles, offsets, t.data_ptr());
115 + TORCH_CHECK(fake_ipc_ptrs.size() == fa->world_size_);
116 + void* ipc_ptrs[8];
117 + for (int i = 0; i < fake_ipc_ptrs.size(); i++) {
118 + ipc_ptrs[i] = reinterpret_cast<void*>(fake_ipc_ptrs[i]);
119 + }
120 + fa->register_buffer(ipc_ptrs);
121 }
122
125 -std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
126 - fptr_t _fa) {
123 +// Use vector<int64_t> to represent byte data for python binding compatibility.
124 +std::tuple<std::vector<int64_t>, std::vector<int64_t>>
125 +get_graph_buffer_ipc_meta(fptr_t _fa) {
126 auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
128 - auto [handle_bytes, offsets] = fa->get_graph_buffer_ipc_meta();
129 - auto options =
130 - torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU);
131 - auto handles =
132 - torch::empty({static_cast<int64_t>(handle_bytes.size())}, options);
133 - std::memcpy(handles.data_ptr(), handle_bytes.data(), handle_bytes.size());
134 - return {handles, std::move(offsets)};
127 + auto [handle, offsets] = fa->get_graph_buffer_ipc_meta();
128 + std::vector<int64_t> bytes(handle.begin(), handle.end());
129 + return std::make_tuple(bytes, offsets);
130 }
131
137 -void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
132 +// Use vector<int64_t> to represent byte data for python binding compatibility.
133 +void register_graph_buffers(fptr_t _fa,
134 + const std::vector<std::vector<int64_t>>& handles,
135 const std::vector<std::vector<int64_t>>& offsets) {
136 auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
140 - fa->register_graph_buffers(handles, offsets);
137 + std::vector<std::string> bytes;
138 + bytes.reserve(handles.size());
139 + for (int i = 0; i < handles.size(); i++) {
140 + bytes.emplace_back(handles[i].begin(), handles[i].end());
141 + }
142 + bytes.reserve(handles.size());
143 + fa->register_graph_buffers(bytes, offsets);
144 }
csrc/custom_all_reduce.cuh
+45 -42
@@ -285,46 +285,52 @@ class CustomAllreduce {
285 int world_size_;
286 bool full_nvlink_;
287
288 - // below are device pointers
288 RankSignals sg_;
289 + // Stores an map from a pointer to its peer pointters from all ranks.
290 std::unordered_map<void*, RankData*> buffers_;
291 Signal* self_sg_;
292
293 - // stores the registered device pointers from all ranks
293 + // Stores rank data from all ranks. This is mainly for cuda graph purposes.
294 + // For cuda graph to work, all kernel arguments must be fixed during graph
295 + // capture time. However, the peer pointers are not known during graph capture
296 + // time. Therefore, during capture, we increment the rank data pointer and use
297 + // that as the argument to the kernel. The kernel arguments are stored in
298 + // graph_unreg_buffers_. The actual peer pointers will be filled in at the
299 + // memory pointed to by the pointers in graph_unreg_buffers_ when
300 + // the IPC handles are exchanged between ranks.
301 + //
302 + // The overall process looks like this:
303 + // 1. Graph capture.
304 + // 2. Each rank obtains the IPC handles for each addresses used during cuda
305 + // graph capture using get_graph_buffer_ipc_meta.
306 + // 3. (In Python) all gather the IPC handles.
307 + // 4. Obtain the peer pointers by opening the IPC handles, and store them in
308 + // the rank data array at corresponding positions.
309 RankData *d_rank_data_base_, *d_rank_data_end_;
310 std::vector<void*> graph_unreg_buffers_;
311 // a map from IPC handles to opened IPC pointers
312 std::map<IPC_KEY, char*> ipc_handles_;
313
314 /**
300 - * meta is a pointer to device metadata and temporary buffer for allreduce.
315 + * Signals are an array of ipc-enabled buffers from all ranks.
316 + * For each of the buffer, the layout is as follows:
317 + * | -- sizeof(Signal) -- | ------ a few MB ----- |
318 + * The first section is for allreduce synchronization, and the second section
319 + * is for storing the intermediate results required by some allreduce algos.
320 *
302 - * There's a total of sizeof(Signal) of prefix before the actual data,
303 - * so meta + 1 points to actual temporary buffer.
304 - *
305 - * note: this class does not own any device memory. Any required buffers
306 - * are passed in from the constructor
321 + * Note: this class does not own any device memory. Any required buffers
322 + * are passed in from the constructor.
323 */
308 - CustomAllreduce(Signal* meta, void* rank_data, size_t rank_data_sz,
309 - const cudaIpcMemHandle_t* handles,
310 - const std::vector<int64_t>& offsets, int rank,
311 - bool full_nvlink = true)
324 + CustomAllreduce(Signal** signals, void* rank_data, size_t rank_data_sz,
325 + int rank, int world_size, bool full_nvlink = true)
326 : rank_(rank),
313 - world_size_(offsets.size()),
327 + world_size_(world_size),
328 full_nvlink_(full_nvlink),
315 - self_sg_(meta),
329 + self_sg_(signals[rank]),
330 d_rank_data_base_(reinterpret_cast<RankData*>(rank_data)),
331 d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
332 for (int i = 0; i < world_size_; i++) {
319 - Signal* rank_sg;
320 - if (i != rank_) {
321 - char* handle = open_ipc_handle(&handles[i]);
322 - handle += offsets[i];
323 - rank_sg = (Signal*)handle;
324 - } else {
325 - rank_sg = self_sg_;
326 - }
327 - sg_.signals[i] = rank_sg;
333 + sg_.signals[i] = signals[i];
334 }
335 }
336
@@ -341,11 +347,10 @@ class CustomAllreduce {
347 return it->second;
348 }
349
344 - std::pair<std::vector<uint8_t>, std::vector<int64_t>>
345 - get_graph_buffer_ipc_meta() {
350 + std::pair<std::string, std::vector<int64_t>> get_graph_buffer_ipc_meta() {
351 auto num_buffers = graph_unreg_buffers_.size();
352 auto handle_sz = sizeof(cudaIpcMemHandle_t);
348 - std::vector<uint8_t> handles(handle_sz * num_buffers, 0);
353 + std::string handles(handle_sz * num_buffers, static_cast<char>(0));
354 std::vector<int64_t> offsets(num_buffers);
355 for (int i = 0; i < num_buffers; i++) {
356 auto ptr = graph_unreg_buffers_[i];
@@ -370,26 +375,22 @@ class CustomAllreduce {
375 std::to_string(d_rank_data_base_ + num - d_rank_data_end_));
376 }
377
373 - void register_buffer(const std::vector<std::string>& handles,
374 - const std::vector<int64_t>& offsets, void* self) {
378 + /**
379 + * Register already-shared IPC pointers.
380 + */
381 + void register_buffer(void** ptrs) {
382 check_rank_data_capacity();
383 RankData data;
384 for (int i = 0; i < world_size_; i++) {
378 - if (i != rank_) {
379 - char* handle = open_ipc_handle(handles[i].data());
380 - handle += offsets[i];
381 - data.ptrs[i] = handle;
382 - } else {
383 - data.ptrs[i] = self;
384 - }
385 + data.ptrs[i] = ptrs[i];
386 }
387 auto d_data = d_rank_data_base_++;
388 CUDACHECK(
389 cudaMemcpy(d_data, &data, sizeof(RankData), cudaMemcpyHostToDevice));
389 - buffers_[self] = d_data;
390 + buffers_[ptrs[rank_]] = d_data;
391 }
392
392 - // note: when registering graph buffers, we intentionally choose to not
393 + // Note: when registering graph buffers, we intentionally choose to not
394 // deduplicate the addresses. That means if the allocator reuses some
395 // addresses, they will be registered again. This is to account for the remote
396 // possibility of different allocation patterns between ranks. For example,
@@ -424,11 +425,13 @@ class CustomAllreduce {
425 }
426
427 /**
427 - * This is the result after careful grid search. Using 36 blocks give the best
428 - * or close to the best runtime on the devices I tried: A100, A10, A30, T4,
429 - * V100. You'll notice that NCCL kernels also only take a small amount of SMs.
430 - * Not quite sure the underlying reason, but my guess is that too many SMs
431 - * will cause contention on NVLink bus.
428 + * Performs allreduce, assuming input has already been registered.
429 + *
430 + * Block and grid default configs are results after careful grid search. Using
431 + * 36 blocks give the best or close to the best runtime on the devices I
432 + * tried: A100, A10, A30, T4, V100. You'll notice that NCCL kernels also only
433 + * take a small amount of SMs. Not quite sure the underlying reason, but my
434 + * guess is that too many SMs will cause contention on NVLink bus.
435 */
436 template <typename T>
437 void allreduce(cudaStream_t stream, T* input, T* output, int size,
csrc/custom_all_reduce_test.cu
+13 -11
@@ -135,24 +135,26 @@ void run(int myRank, int nRanks, ncclComm_t& comm, int threads, int block_limit,
135 void* rank_data;
136 size_t rank_data_sz = 16 * 1024 * 1024;
137 CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
138 - std::vector<int64_t> offsets(nRanks, 0);
139 - vllm::CustomAllreduce fa(buffer, rank_data, rank_data_sz, data_handles,
140 - offsets, myRank);
138 + vllm::Signal* ipc_ptrs[8];
139 + for (int i = 0; i < nRanks; i++) {
140 + if (i == myRank)
141 + ipc_ptrs[i] = buffer;
142 + else
143 + CUDACHECK(cudaIpcOpenMemHandle((void**)&ipc_ptrs[i], data_handles[i],
144 + cudaIpcMemLazyEnablePeerAccess));
145 + }
146 + vllm::CustomAllreduce fa(ipc_ptrs, rank_data, rank_data_sz, myRank, nRanks);
147 auto* self_data =
148 reinterpret_cast<T*>(reinterpret_cast<char*>(buffer) +
149 sizeof(vllm::Signal) + data_size * sizeof(T));
150 // hack buffer registration
151 {
146 - std::vector<std::string> handles;
147 - handles.reserve(nRanks);
152 + void* data[8];
153 for (int i = 0; i < nRanks; i++) {
149 - char* begin = (char*)&data_handles[i];
150 - char* end = (char*)&data_handles[i + 1];
151 - handles.emplace_back(begin, end);
154 + data[i] =
155 + ((char*)ipc_ptrs[i]) + sizeof(vllm::Signal) + data_size * sizeof(T);
156 }
153 - std::vector<int64_t> offsets(nRanks,
154 - sizeof(vllm::Signal) + data_size * sizeof(T));
155 - fa.register_buffer(handles, offsets, self_data);
157 + fa.register_buffer(data);
158 }
159
160 double* ground_truth;
csrc/ops.h
+9 -13
@@ -199,20 +199,16 @@ void causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
199
200 #ifndef USE_ROCM
201 using fptr_t = int64_t;
202 -fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
203 - const std::vector<std::string>& handles,
204 - const std::vector<int64_t>& offsets, int64_t rank,
205 - bool full_nvlink);
206 -void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out);
207 -void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
208 - torch::Tensor& out);
202 +fptr_t init_custom_ar(const std::vector<int64_t>& fake_ipc_ptrs,
203 + torch::Tensor& rank_data, int64_t rank, bool full_nvlink);
204 +void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
205 + fptr_t reg_buffer, int64_t reg_buffer_sz_bytes);
206 void dispose(fptr_t _fa);
207 int64_t meta_size();
211 -void register_buffer(fptr_t _fa, torch::Tensor& t,
212 - const std::vector<std::string>& handles,
213 - const std::vector<int64_t>& offsets);
214 -std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
215 - fptr_t _fa);
216 -void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
208 +void register_buffer(fptr_t _fa, const std::vector<int64_t>& fake_ipc_ptrs);
209 +std::tuple<std::vector<int64_t>, std::vector<int64_t>>
210 +get_graph_buffer_ipc_meta(fptr_t _fa);
211 +void register_graph_buffers(fptr_t _fa,
212 + const std::vector<std::vector<int64_t>>& handles,
213 const std::vector<std::vector<int64_t>>& offsets);
214 #endif
csrc/torch_bindings.cpp
+6 -15
@@ -411,27 +411,18 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cuda_utils), cuda_utils) {
411 TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
412 // Custom all-reduce kernels
413 custom_ar.def(
414 - "init_custom_ar(Tensor meta, Tensor rank_data, "
415 - "str[] handles, int[] offsets, int rank, "
416 - "bool full_nvlink) -> int");
414 + "init_custom_ar(int[] ipc_tensors, Tensor rank_data, "
415 + "int rank, bool full_nvlink) -> int");
416 custom_ar.impl("init_custom_ar", torch::kCUDA, &init_custom_ar);
418 -
419 - custom_ar.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()");
420 - custom_ar.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg);
421 -
417 custom_ar.def(
423 - "all_reduce_unreg(int fa, Tensor inp, Tensor reg_buffer, Tensor! out) -> "
424 - "()");
425 - custom_ar.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg);
418 + "all_reduce(int fa, Tensor inp, Tensor! out, int reg_buffer, "
419 + "int reg_buffer_sz_bytes) -> ()");
420 + custom_ar.impl("all_reduce", torch::kCUDA, &all_reduce);
421
422 custom_ar.def("dispose", &dispose);
423 custom_ar.def("meta_size", &meta_size);
424
430 - custom_ar.def(
431 - "register_buffer(int fa, Tensor t, str[] handles, "
432 - "int[] offsets) -> ()");
433 - custom_ar.impl("register_buffer", torch::kCUDA, &register_buffer);
434 -
425 + custom_ar.def("register_buffer", &register_buffer);
426 custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta);
427 custom_ar.def("register_graph_buffers", &register_graph_buffers);
428 }
tests/distributed/test_custom_all_reduce.py
+2 -2
@@ -95,13 +95,13 @@ def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
95 inp = torch.ones(sz, dtype=torch.float32, device=device)
96 out = inp
97 for _ in range(num_communication):
98 - out = fa.all_reduce_unreg(out)
98 + out = fa.all_reduce(out, registered=False)
99 torch.testing.assert_close(out, inp * (tp_size**num_communication))
100
101 inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
102 out = inp
103 for _ in range(num_communication):
104 - out = fa.all_reduce_unreg(out)
104 + out = fa.all_reduce(out, registered=False)
105 torch.testing.assert_close(out, inp * (tp_size**num_communication))
106
107
tools/profiler/visualize_layerwise_profile.py
+16 -16
@@ -196,8 +196,8 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
196 def is_cross_device_reduce_2stage(op_name: str):
197 return "cross_device_reduce_2stage" in op_name
198
199 - def is_custom_ar_all_reduce_unreg(op_name: str):
200 - return "_C_custom_ar::all_reduce_unreg" in op_name
199 + def is_custom_ar_all_reduce(op_name: str):
200 + return "_C_custom_ar::all_reduce" in op_name
201
202 def is_reduce_kernel(op_name: str):
203 return "reduce_kernel" in op_name
@@ -246,9 +246,9 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
246 filter(lambda x: is_cross_device_reduce_2stage(x), ops))
247 ops = list(filter(lambda x: x not in cross_device_reduce_2stage_ops, ops))
248
249 - custom_ar_all_reduce_unreg_ops = list(
250 - filter(lambda x: is_custom_ar_all_reduce_unreg(x), ops))
251 - ops = list(filter(lambda x: x not in custom_ar_all_reduce_unreg_ops, ops))
249 + custom_ar_all_reduce_ops = list(
250 + filter(lambda x: is_custom_ar_all_reduce(x), ops))
251 + ops = list(filter(lambda x: x not in custom_ar_all_reduce_ops, ops))
252
253 reduce_kernel_ops = list(filter(lambda x: is_reduce_kernel(x), ops))
254 ops = list(filter(lambda x: x not in reduce_kernel_ops, ops))
@@ -289,21 +289,21 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
289 if len(cross_device_reduce_2stage_ops):
290 trace_df['cross_device_reduce_2stage_ops'] = trace_df[
291 cross_device_reduce_2stage_ops].agg("sum", axis=1)
292 - if len(custom_ar_all_reduce_unreg_ops):
293 - trace_df['custom_ar_all_reduce_unreg_ops'] = trace_df[
294 - custom_ar_all_reduce_unreg_ops].agg("sum", axis=1)
292 + if len(custom_ar_all_reduce_ops):
293 + trace_df['custom_ar_all_reduce_ops'] = trace_df[
294 + custom_ar_all_reduce_ops].agg("sum", axis=1)
295 if len(reduce_kernel_ops):
296 trace_df['reduce_kernel_ops'] = trace_df[reduce_kernel_ops].agg("sum",
297 axis=1)
298
299 - trace_df.drop(
300 - attention_ops + quant_ops + gemm_ops + rms_norm_ops + vocab_embed_ops +
301 - mem_ops + elementwise_ops + nccl_all_reduce_ops + nccl_gather_ops +
302 - nccl_broadcast_ops + nccl_other_ops + cross_device_reduce_1stage_ops +
303 - cross_device_reduce_2stage_ops + custom_ar_all_reduce_unreg_ops +
304 - reduce_kernel_ops,
305 - axis=1,
306 - inplace=True)
299 + trace_df.drop(attention_ops + quant_ops + gemm_ops + rms_norm_ops +
300 + vocab_embed_ops + mem_ops + elementwise_ops +
301 + nccl_all_reduce_ops + nccl_gather_ops + nccl_broadcast_ops +
302 + nccl_other_ops + cross_device_reduce_1stage_ops +
303 + cross_device_reduce_2stage_ops + custom_ar_all_reduce_ops +
304 + reduce_kernel_ops,
305 + axis=1,
306 + inplace=True)
307 return trace_df
308
309
vllm/_custom_ops.py
+12 -17
@@ -912,20 +912,16 @@ def get_max_shared_memory_per_block_device_attribute(device: int) -> int:
912
913
914 # custom ar
915 -def init_custom_ar(meta: torch.Tensor, rank_data: torch.Tensor,
916 - handles: List[str], offsets: List[int], rank: int,
917 - full_nvlink: bool) -> int:
918 - return torch.ops._C_custom_ar.init_custom_ar(meta, rank_data, handles,
919 - offsets, rank, full_nvlink)
915 +def init_custom_ar(ipc_tensors: List[torch.Tensor], rank_data: torch.Tensor,
916 + rank: int, full_nvlink: bool) -> int:
917 + return torch.ops._C_custom_ar.init_custom_ar(ipc_tensors, rank_data, rank,
918 + full_nvlink)
919
920
922 -def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None:
923 - torch.ops._C_custom_ar.all_reduce_reg(fa, inp, out)
924 -
925 -
926 -def all_reduce_unreg(fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor,
927 - out: torch.Tensor) -> None:
928 - torch.ops._C_custom_ar.all_reduce_unreg(fa, inp, reg_buffer, out)
921 +def all_reduce(fa: int, inp: torch.Tensor, out: torch.Tensor, reg_buffer: int,
922 + reg_buffer_sz_bytes: int) -> None:
923 + torch.ops._C_custom_ar.all_reduce(fa, inp, out, reg_buffer,
924 + reg_buffer_sz_bytes)
925
926
927 def dispose(fa: int) -> None:
@@ -936,16 +932,15 @@ def meta_size() -> int:
932 return torch.ops._C_custom_ar.meta_size()
933
934
939 -def register_buffer(fa: int, t: torch.Tensor, handles: List[str],
940 - offsets: List[int]) -> None:
941 - return torch.ops._C_custom_ar.register_buffer(fa, t, handles, offsets)
935 +def register_buffer(fa: int, ipc_tensors: List[int]) -> None:
936 + return torch.ops._C_custom_ar.register_buffer(fa, ipc_tensors)
937
938
944 -def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[str], List[int]]:
939 +def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[int], List[int]]:
940 return torch.ops._C_custom_ar.get_graph_buffer_ipc_meta(fa)
941
942
948 -def register_graph_buffers(fa: int, handles: List[str],
943 +def register_graph_buffers(fa: int, handles: List[List[int]],
944 offsets: List[List[int]]) -> None:
945 torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets)
946
vllm/distributed/device_communicators/custom_all_reduce.py
+54 -86
@@ -1,6 +1,6 @@
1 import ctypes
2 from contextlib import contextmanager
3 -from typing import Any, List, Optional, Union
3 +from typing import List, Optional, Union
4
5 import torch
6 import torch.distributed as dist
@@ -147,18 +147,14 @@ class CustomAllreduce:
147 return
148
149 self.disabled = False
150 - # buffers memory are owned by this Python class and passed to C++
151 - # meta data composes of two parts: meta data for synchronization
152 - # (256 bytes) and a temporary buffer for storing intermediate
153 - # allreduce results.
154 - self.meta = torch.zeros(ops.meta_size() + max_size,
155 - dtype=torch.uint8,
156 - device=self.device)
150 + # Buffers memory are owned by this Python class and passed to C++.
151 + # Meta data composes of two parts: meta data for synchronization and a
152 + # temporary buffer for storing intermediate allreduce results.
153 + self.meta_ptrs = self.create_shared_buffer(ops.meta_size() + max_size,
154 + group=group)
155 # This is a pre-registered IPC buffer. In eager mode, input tensors
156 # are first copied into this buffer before allreduce is performed
159 - self.buffer = torch.empty(max_size,
160 - dtype=torch.uint8,
161 - device=self.device)
157 + self.buffer_ptrs = self.create_shared_buffer(max_size, group=group)
158 # This is a buffer for storing the tuples of pointers pointing to
159 # IPC buffers from all ranks. Each registered tuple has size of
160 # 8*world_size bytes where world_size is at most 8. Allocating 8MB
@@ -170,16 +166,19 @@ class CustomAllreduce:
166 self.max_size = max_size
167 self.rank = rank
168 self.world_size = world_size
173 - handles, offsets = self._get_ipc_meta(self.meta)
169 self.full_nvlink = full_nvlink
175 - self._ptr = ops.init_custom_ar(self.meta, self.rank_data, handles,
176 - offsets, rank, self.full_nvlink)
177 - self.register_buffer(self.buffer)
170 + self._ptr = ops.init_custom_ar(self.meta_ptrs, self.rank_data, rank,
171 + self.full_nvlink)
172 + ops.register_buffer(self._ptr, self.buffer_ptrs)
173
174 @staticmethod
175 def create_shared_buffer(
176 size_in_bytes: int,
177 group: Optional[ProcessGroup] = None) -> List[int]:
178 + """
179 + Creates a shared buffer and returns a list of pointers
180 + representing the buffer on all processes in the group.
181 + """
182 lib = CudaRTLibrary()
183 pointer = lib.cudaMalloc(size_in_bytes)
184 handle = lib.cudaIpcGetMemHandle(pointer)
@@ -220,60 +219,24 @@ class CustomAllreduce:
219 if not self.disabled:
220 self.register_graph_buffers()
221
223 - def _get_ipc_meta(self, inp: torch.Tensor):
224 - data = inp.untyped_storage()._share_cuda_()
225 - handle = data[1]
226 - # https://github.com/pytorch/pytorch/pull/130890 changes
227 - # the binary format of the ipc handle
228 - # it starts from pytorch 2.5
229 - if len(handle) > 64:
230 - assert len(handle) == 66
231 - # only support SHAREABLE_HANDLE_VERSION = 1
232 - assert int(handle[0]) == 1
233 - # only support SHAREABLE_CUDA_MALLOC = 'c'
234 - assert handle[1] == ord("c")
235 - handle = handle[2:]
236 - # TODO: support expandable segment
237 - shard_data = (
238 - handle, # ipc handle to base ptr
239 - data[3], # offset of base ptr
240 - )
241 - return self._gather_ipc_meta(shard_data)
242 -
243 - def _gather_ipc_meta(self, shard_data):
244 - # Note: don't use `[[None]] * self.world_size` here
245 - # because it will create a list of the same reference
246 - all_data: List[Optional[Any]] = [[None]
247 - for i in range(self.world_size)]
248 - all_data[self.rank][0] = shard_data
249 -
250 - ranks = dist.get_process_group_ranks(group=self.group)
251 - ranks.sort()
222 + def register_graph_buffers(self):
223 + handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr)
224 + logger.info("Registering %d cuda graph addresses", len(offset))
225 + # We cannot directly use `dist.all_gather_object` here
226 + # because it is incompatible with `gloo` backend under inference mode.
227 + # see https://github.com/pytorch/pytorch/issues/126032 for details.
228 + all_data = [[None, None]
229 + for _ in range(dist.get_world_size(group=self.group))]
230 + all_data[self.rank] = [handle, offset]
231 + ranks = sorted(dist.get_process_group_ranks(group=self.group))
232 for i, rank in enumerate(ranks):
233 dist.broadcast_object_list(all_data[i],
234 src=rank,
235 group=self.group,
236 device="cpu")
257 -
258 - # we cannot directly use `dist.all_gather_object` here
259 - # because it is incompatible with `gloo` backend under inference mode.
260 - # see https://github.com/pytorch/pytorch/issues/126032 for details.
261 -
262 - handles = []
263 - offsets = []
264 - for i in range(len(all_data)):
265 - handles.append(all_data[i][0][0]) # type: ignore
266 - offsets.append(all_data[i][0][1]) # type: ignore
267 - return handles, offsets
268 -
269 - def register_buffer(self, inp: torch.Tensor):
270 - handles, offsets = self._get_ipc_meta(inp)
271 - ops.register_buffer(self._ptr, inp, handles, offsets)
272 -
273 - def register_graph_buffers(self):
274 - handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr)
275 - handles, offsets = self._gather_ipc_meta((bytes(handle), offset))
276 - logger.info("Registering %d cuda graph addresses", len(offset))
237 + # Unpack list of tuples to tuple of lists.
238 + handles = [d[0] for d in all_data] # type: ignore
239 + offsets = [d[1] for d in all_data] # type: ignore
240 ops.register_graph_buffers(self._ptr, handles, offsets)
241
242 def should_custom_ar(self, inp: torch.Tensor):
@@ -291,45 +254,50 @@ class CustomAllreduce:
254 return inp_size < self.max_size
255 return False
256
294 - # all reduce, assuming inp tensor is IPC registered with register_buffer,
295 - # or, in the context of cuda graphs, register_graph_buffers
296 - def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None):
297 - if out is None:
298 - out = torch.empty_like(inp)
299 - ops.all_reduce_reg(self._ptr, inp, out)
300 - return out
301 -
302 - # all reduce, assuming inp tensor is NOT IPC registered
303 - def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None):
257 + def all_reduce(self,
258 + inp: torch.Tensor,
259 + *,
260 + out: torch.Tensor = None,
261 + registered: bool = False):
262 + """Performs an out-of-place all reduce.
263 +
264 + If registered is True, this assumes inp's pointer is already
265 + IPC-registered. Otherwise, inp is first copied into a pre-registered
266 + buffer.
267 + """
268 if out is None:
269 out = torch.empty_like(inp)
306 - ops.all_reduce_unreg(self._ptr, inp, self.buffer, out)
270 + if registered:
271 + ops.all_reduce(self._ptr, inp, out, 0, 0)
272 + else:
273 + ops.all_reduce(self._ptr, inp, out, self.buffer_ptrs[self.rank],
274 + self.max_size)
275 return out
276
277 def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
310 - # when custom allreduce is disabled, this will be None
278 + """The main allreduce API that provides support for cuda graph."""
279 + # When custom allreduce is disabled, this will be None.
280 if self.disabled or not self.should_custom_ar(input):
281 return None
282 if self._IS_CAPTURING:
283 if torch.cuda.is_current_stream_capturing():
315 - return self.all_reduce_reg(input)
284 + return self.all_reduce(input, registered=True)
285 else:
317 - # if warm up, mimic the allocation pattern
318 - # since custom allreduce is out-of-place
286 + # If warm up, mimic the allocation pattern since custom
287 + # allreduce is out-of-place.
288 return torch.empty_like(input)
289 else:
321 - # note: outside of cuda graph context,
322 - # custom allreduce incurs a cost of cudaMemcpy, which should
323 - # be small(<=1% of overall latency) compared to the performance
324 - # gains of using custom kernels
325 - return self.all_reduce_unreg(input)
326 -
327 - return None
290 + # Note: outside of cuda graph context, custom allreduce incurs a
291 + # cost of cudaMemcpy, which should be small (<=1% of overall
292 + # latency) compared to the performance gain of using custom kernels
293 + return self.all_reduce(input, registered=False)
294
295 def close(self):
296 if not self.disabled and self._ptr:
297 ops.dispose(self._ptr)
298 self._ptr = 0
299 + self.free_shared_buffer(self.meta_ptrs)
300 + self.free_shared_buffer(self.buffer_ptrs)
301
302 def __del__(self):
303 self.close()