A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Core][Distributed] Refactor ipc buffer init in CustomAllreduce (#10030)
Signed-off-by: Hanzhi Zhou <hanzhi713@gmail.com>
Hanzhi Zhou committed
Nov 6, 2024 at 23:50 UTC
6192e9b8fef8492c3e52bd65c7d954a1ef9b40c8
9 files changed
+218
-260
csrc/custom_all_reduce.cu
+61
-58
@@ -5,32 +5,29 @@
5
6
#include "custom_all_reduce.cuh"
7
8
-// fake pointer type, must match fptr_t type in ops.h
8
+// Fake pointer type, must match fptr_t type in ops.h.
9
+// We use this type alias to indicate when pointers are passed in as int64_t.
10
using fptr_t = int64_t;
11
static_assert(sizeof(void*) == sizeof(fptr_t));
12
12
-fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
13
- const std::vector<std::string>& handles,
14
- const std::vector<int64_t>& offsets, int64_t rank,
13
+fptr_t init_custom_ar(const std::vector<fptr_t>& fake_ipc_ptrs,
14
+ torch::Tensor& rank_data, int64_t rank,
15
bool full_nvlink) {
16
- int world_size = offsets.size();
16
+ int world_size = fake_ipc_ptrs.size();
17
if (world_size > 8)
18
throw std::invalid_argument("world size > 8 is not supported");
19
if (world_size % 2 != 0)
20
throw std::invalid_argument("Odd num gpus is not supported for now");
21
- if (world_size != handles.size())
22
- throw std::invalid_argument(
23
- "handles length should equal to offsets length");
21
if (rank < 0 || rank >= world_size)
22
throw std::invalid_argument("invalid rank passed in");
23
27
- cudaIpcMemHandle_t ipc_handles[8];
24
+ vllm::Signal* ipc_ptrs[8];
25
for (int i = 0; i < world_size; i++) {
29
- std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
26
+ ipc_ptrs[i] = reinterpret_cast<vllm::Signal*>(fake_ipc_ptrs[i]);
27
}
31
- return (fptr_t) new vllm::CustomAllreduce(
32
- reinterpret_cast<vllm::Signal*>(meta.data_ptr()), rank_data.data_ptr(),
33
- rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
28
+ return (fptr_t) new vllm::CustomAllreduce(ipc_ptrs, rank_data.data_ptr(),
29
+ rank_data.numel(), rank, world_size,
30
+ full_nvlink);
31
}
32
33
/**
@@ -55,26 +52,48 @@ bool _is_weak_contiguous(torch::Tensor& t) {
52
t.numel() * t.element_size());
53
}
54
58
-void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
59
- cudaStream_t stream) {
55
+/**
56
+ * Performs an out-of-place allreduce and stores result in out.
57
+ *
58
+ * If _reg_buffer is null, assumes inp.data_ptr() is already IPC-registered.
59
+ * Otherwise, _reg_buffer is assumed to be IPC-registered and inp is first
60
+ * copied into _reg_buffer.
61
+ */
62
+void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
63
+ fptr_t _reg_buffer, int64_t reg_buffer_sz_bytes) {
64
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
65
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
66
+ auto stream = c10::cuda::getCurrentCUDAStream().stream();
67
+
68
+ TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
69
+ TORCH_CHECK_EQ(inp.numel(), out.numel());
70
TORCH_CHECK(_is_weak_contiguous(out));
71
+ TORCH_CHECK(_is_weak_contiguous(inp));
72
+ auto input_size = inp.numel() * inp.element_size();
73
+ auto reg_buffer = reinterpret_cast<void*>(_reg_buffer);
74
+ if (reg_buffer) {
75
+ TORCH_CHECK_LE(input_size, reg_buffer_sz_bytes);
76
+ AT_CUDA_CHECK(cudaMemcpyAsync(reg_buffer, inp.data_ptr(), input_size,
77
+ cudaMemcpyDeviceToDevice, stream));
78
+ } else {
79
+ reg_buffer = inp.data_ptr();
80
+ }
81
switch (out.scalar_type()) {
82
case at::ScalarType::Float: {
64
- fa->allreduce<float>(stream, reinterpret_cast<float*>(inp.data_ptr()),
83
+ fa->allreduce<float>(stream, reinterpret_cast<float*>(reg_buffer),
84
reinterpret_cast<float*>(out.data_ptr()),
85
out.numel());
86
break;
87
}
88
case at::ScalarType::Half: {
70
- fa->allreduce<half>(stream, reinterpret_cast<half*>(inp.data_ptr()),
89
+ fa->allreduce<half>(stream, reinterpret_cast<half*>(reg_buffer),
90
reinterpret_cast<half*>(out.data_ptr()), out.numel());
91
break;
92
}
93
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
94
case at::ScalarType::BFloat16: {
95
fa->allreduce<nv_bfloat16>(
77
- stream, reinterpret_cast<nv_bfloat16*>(inp.data_ptr()),
96
+ stream, reinterpret_cast<nv_bfloat16*>(reg_buffer),
97
reinterpret_cast<nv_bfloat16*>(out.data_ptr()), out.numel());
98
break;
99
}
@@ -85,57 +104,41 @@ void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
104
}
105
}
106
88
-void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out) {
89
- const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
90
- auto stream = c10::cuda::getCurrentCUDAStream().stream();
91
- TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
92
- TORCH_CHECK_EQ(inp.numel(), out.numel());
93
- _all_reduce(_fa, inp, out, stream);
94
-}
95
-
96
-void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
97
- torch::Tensor& out) {
98
- const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
99
- auto stream = c10::cuda::getCurrentCUDAStream().stream();
100
-
101
- auto input_size = inp.numel() * inp.element_size();
102
- TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
103
- TORCH_CHECK_EQ(inp.numel(), out.numel());
104
- TORCH_CHECK(input_size <= reg_buffer.numel() * reg_buffer.element_size(),
105
- "registered buffer is too small to contain the input");
106
- AT_CUDA_CHECK(cudaMemcpyAsync(reg_buffer.data_ptr(), inp.data_ptr(),
107
- input_size, cudaMemcpyDeviceToDevice, stream));
108
- _all_reduce(_fa, reg_buffer, out, stream);
109
-}
110
-
107
void dispose(fptr_t _fa) {
112
- auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
113
- delete fa;
108
+ delete reinterpret_cast<vllm::CustomAllreduce*>(_fa);
109
}
110
111
int64_t meta_size() { return sizeof(vllm::Signal); }
112
118
-void register_buffer(fptr_t _fa, torch::Tensor& t,
119
- const std::vector<std::string>& handles,
120
- const std::vector<int64_t>& offsets) {
113
+void register_buffer(fptr_t _fa, const std::vector<fptr_t>& fake_ipc_ptrs) {
114
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
122
- fa->register_buffer(handles, offsets, t.data_ptr());
115
+ TORCH_CHECK(fake_ipc_ptrs.size() == fa->world_size_);
116
+ void* ipc_ptrs[8];
117
+ for (int i = 0; i < fake_ipc_ptrs.size(); i++) {
118
+ ipc_ptrs[i] = reinterpret_cast<void*>(fake_ipc_ptrs[i]);
119
+ }
120
+ fa->register_buffer(ipc_ptrs);
121
}
122
125
-std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
126
- fptr_t _fa) {
123
+// Use vector<int64_t> to represent byte data for python binding compatibility.
124
+std::tuple<std::vector<int64_t>, std::vector<int64_t>>
125
+get_graph_buffer_ipc_meta(fptr_t _fa) {
126
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
128
- auto [handle_bytes, offsets] = fa->get_graph_buffer_ipc_meta();
129
- auto options =
130
- torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU);
131
- auto handles =
132
- torch::empty({static_cast<int64_t>(handle_bytes.size())}, options);
133
- std::memcpy(handles.data_ptr(), handle_bytes.data(), handle_bytes.size());
134
- return {handles, std::move(offsets)};
127
+ auto [handle, offsets] = fa->get_graph_buffer_ipc_meta();
128
+ std::vector<int64_t> bytes(handle.begin(), handle.end());
129
+ return std::make_tuple(bytes, offsets);
130
}
131
137
-void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
132
+// Use vector<int64_t> to represent byte data for python binding compatibility.
133
+void register_graph_buffers(fptr_t _fa,
134
+ const std::vector<std::vector<int64_t>>& handles,
135
const std::vector<std::vector<int64_t>>& offsets) {
136
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
140
- fa->register_graph_buffers(handles, offsets);
137
+ std::vector<std::string> bytes;
138
+ bytes.reserve(handles.size());
139
+ for (int i = 0; i < handles.size(); i++) {
140
+ bytes.emplace_back(handles[i].begin(), handles[i].end());
141
+ }
142
+ bytes.reserve(handles.size());
143
+ fa->register_graph_buffers(bytes, offsets);
144
}
csrc/custom_all_reduce.cuh
+45
-42
@@ -285,46 +285,52 @@ class CustomAllreduce {
285
int world_size_;
286
bool full_nvlink_;
287
288
- // below are device pointers
288
RankSignals sg_;
289
+ // Stores an map from a pointer to its peer pointters from all ranks.
290
std::unordered_map<void*, RankData*> buffers_;
291
Signal* self_sg_;
292
293
- // stores the registered device pointers from all ranks
293
+ // Stores rank data from all ranks. This is mainly for cuda graph purposes.
294
+ // For cuda graph to work, all kernel arguments must be fixed during graph
295
+ // capture time. However, the peer pointers are not known during graph capture
296
+ // time. Therefore, during capture, we increment the rank data pointer and use
297
+ // that as the argument to the kernel. The kernel arguments are stored in
298
+ // graph_unreg_buffers_. The actual peer pointers will be filled in at the
299
+ // memory pointed to by the pointers in graph_unreg_buffers_ when
300
+ // the IPC handles are exchanged between ranks.
301
+ //
302
+ // The overall process looks like this:
303
+ // 1. Graph capture.
304
+ // 2. Each rank obtains the IPC handles for each addresses used during cuda
305
+ // graph capture using get_graph_buffer_ipc_meta.
306
+ // 3. (In Python) all gather the IPC handles.
307
+ // 4. Obtain the peer pointers by opening the IPC handles, and store them in
308
+ // the rank data array at corresponding positions.
309
RankData *d_rank_data_base_, *d_rank_data_end_;
310
std::vector<void*> graph_unreg_buffers_;
311
// a map from IPC handles to opened IPC pointers
312
std::map<IPC_KEY, char*> ipc_handles_;
313
314
/**
300
- * meta is a pointer to device metadata and temporary buffer for allreduce.
315
+ * Signals are an array of ipc-enabled buffers from all ranks.
316
+ * For each of the buffer, the layout is as follows:
317
+ * | -- sizeof(Signal) -- | ------ a few MB ----- |
318
+ * The first section is for allreduce synchronization, and the second section
319
+ * is for storing the intermediate results required by some allreduce algos.
320
*
302
- * There's a total of sizeof(Signal) of prefix before the actual data,
303
- * so meta + 1 points to actual temporary buffer.
304
- *
305
- * note: this class does not own any device memory. Any required buffers
306
- * are passed in from the constructor
321
+ * Note: this class does not own any device memory. Any required buffers
322
+ * are passed in from the constructor.
323
*/
308
- CustomAllreduce(Signal* meta, void* rank_data, size_t rank_data_sz,
309
- const cudaIpcMemHandle_t* handles,
310
- const std::vector<int64_t>& offsets, int rank,
311
- bool full_nvlink = true)
324
+ CustomAllreduce(Signal** signals, void* rank_data, size_t rank_data_sz,
325
+ int rank, int world_size, bool full_nvlink = true)
326
: rank_(rank),
313
- world_size_(offsets.size()),
327
+ world_size_(world_size),
328
full_nvlink_(full_nvlink),
315
- self_sg_(meta),
329
+ self_sg_(signals[rank]),
330
d_rank_data_base_(reinterpret_cast<RankData*>(rank_data)),
331
d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
332
for (int i = 0; i < world_size_; i++) {
319
- Signal* rank_sg;
320
- if (i != rank_) {
321
- char* handle = open_ipc_handle(&handles[i]);
322
- handle += offsets[i];
323
- rank_sg = (Signal*)handle;
324
- } else {
325
- rank_sg = self_sg_;
326
- }
327
- sg_.signals[i] = rank_sg;
333
+ sg_.signals[i] = signals[i];
334
}
335
}
336
@@ -341,11 +347,10 @@ class CustomAllreduce {
347
return it->second;
348
}
349
344
- std::pair<std::vector<uint8_t>, std::vector<int64_t>>
345
- get_graph_buffer_ipc_meta() {
350
+ std::pair<std::string, std::vector<int64_t>> get_graph_buffer_ipc_meta() {
351
auto num_buffers = graph_unreg_buffers_.size();
352
auto handle_sz = sizeof(cudaIpcMemHandle_t);
348
- std::vector<uint8_t> handles(handle_sz * num_buffers, 0);
353
+ std::string handles(handle_sz * num_buffers, static_cast<char>(0));
354
std::vector<int64_t> offsets(num_buffers);
355
for (int i = 0; i < num_buffers; i++) {
356
auto ptr = graph_unreg_buffers_[i];
@@ -370,26 +375,22 @@ class CustomAllreduce {
375
std::to_string(d_rank_data_base_ + num - d_rank_data_end_));
376
}
377
373
- void register_buffer(const std::vector<std::string>& handles,
374
- const std::vector<int64_t>& offsets, void* self) {
378
+ /**
379
+ * Register already-shared IPC pointers.
380
+ */
381
+ void register_buffer(void** ptrs) {
382
check_rank_data_capacity();
383
RankData data;
384
for (int i = 0; i < world_size_; i++) {
378
- if (i != rank_) {
379
- char* handle = open_ipc_handle(handles[i].data());
380
- handle += offsets[i];
381
- data.ptrs[i] = handle;
382
- } else {
383
- data.ptrs[i] = self;
384
- }
385
+ data.ptrs[i] = ptrs[i];
386
}
387
auto d_data = d_rank_data_base_++;
388
CUDACHECK(
389
cudaMemcpy(d_data, &data, sizeof(RankData), cudaMemcpyHostToDevice));
389
- buffers_[self] = d_data;
390
+ buffers_[ptrs[rank_]] = d_data;
391
}
392
392
- // note: when registering graph buffers, we intentionally choose to not
393
+ // Note: when registering graph buffers, we intentionally choose to not
394
// deduplicate the addresses. That means if the allocator reuses some
395
// addresses, they will be registered again. This is to account for the remote
396
// possibility of different allocation patterns between ranks. For example,
@@ -424,11 +425,13 @@ class CustomAllreduce {
425
}
426
427
/**
427
- * This is the result after careful grid search. Using 36 blocks give the best
428
- * or close to the best runtime on the devices I tried: A100, A10, A30, T4,
429
- * V100. You'll notice that NCCL kernels also only take a small amount of SMs.
430
- * Not quite sure the underlying reason, but my guess is that too many SMs
431
- * will cause contention on NVLink bus.
428
+ * Performs allreduce, assuming input has already been registered.
429
+ *
430
+ * Block and grid default configs are results after careful grid search. Using
431
+ * 36 blocks give the best or close to the best runtime on the devices I
432
+ * tried: A100, A10, A30, T4, V100. You'll notice that NCCL kernels also only
433
+ * take a small amount of SMs. Not quite sure the underlying reason, but my
434
+ * guess is that too many SMs will cause contention on NVLink bus.
435
*/
436
template <typename T>
437
void allreduce(cudaStream_t stream, T* input, T* output, int size,
csrc/custom_all_reduce_test.cu
+13
-11
@@ -135,24 +135,26 @@ void run(int myRank, int nRanks, ncclComm_t& comm, int threads, int block_limit,
135
void* rank_data;
136
size_t rank_data_sz = 16 * 1024 * 1024;
137
CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
138
- std::vector<int64_t> offsets(nRanks, 0);
139
- vllm::CustomAllreduce fa(buffer, rank_data, rank_data_sz, data_handles,
140
- offsets, myRank);
138
+ vllm::Signal* ipc_ptrs[8];
139
+ for (int i = 0; i < nRanks; i++) {
140
+ if (i == myRank)
141
+ ipc_ptrs[i] = buffer;
142
+ else
143
+ CUDACHECK(cudaIpcOpenMemHandle((void**)&ipc_ptrs[i], data_handles[i],
144
+ cudaIpcMemLazyEnablePeerAccess));
145
+ }
146
+ vllm::CustomAllreduce fa(ipc_ptrs, rank_data, rank_data_sz, myRank, nRanks);
147
auto* self_data =
148
reinterpret_cast<T*>(reinterpret_cast<char*>(buffer) +
149
sizeof(vllm::Signal) + data_size * sizeof(T));
150
// hack buffer registration
151
{
146
- std::vector<std::string> handles;
147
- handles.reserve(nRanks);
152
+ void* data[8];
153
for (int i = 0; i < nRanks; i++) {
149
- char* begin = (char*)&data_handles[i];
150
- char* end = (char*)&data_handles[i + 1];
151
- handles.emplace_back(begin, end);
154
+ data[i] =
155
+ ((char*)ipc_ptrs[i]) + sizeof(vllm::Signal) + data_size * sizeof(T);
156
}
153
- std::vector<int64_t> offsets(nRanks,
154
- sizeof(vllm::Signal) + data_size * sizeof(T));
155
- fa.register_buffer(handles, offsets, self_data);
157
+ fa.register_buffer(data);
158
}
159
160
double* ground_truth;
csrc/ops.h
+9
-13
@@ -199,20 +199,16 @@ void causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
199
200
#ifndef USE_ROCM
201
using fptr_t = int64_t;
202
-fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
203
- const std::vector<std::string>& handles,
204
- const std::vector<int64_t>& offsets, int64_t rank,
205
- bool full_nvlink);
206
-void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out);
207
-void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
208
- torch::Tensor& out);
202
+fptr_t init_custom_ar(const std::vector<int64_t>& fake_ipc_ptrs,
203
+ torch::Tensor& rank_data, int64_t rank, bool full_nvlink);
204
+void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
205
+ fptr_t reg_buffer, int64_t reg_buffer_sz_bytes);
206
void dispose(fptr_t _fa);
207
int64_t meta_size();
211
-void register_buffer(fptr_t _fa, torch::Tensor& t,
212
- const std::vector<std::string>& handles,
213
- const std::vector<int64_t>& offsets);
214
-std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
215
- fptr_t _fa);
216
-void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
208
+void register_buffer(fptr_t _fa, const std::vector<int64_t>& fake_ipc_ptrs);
209
+std::tuple<std::vector<int64_t>, std::vector<int64_t>>
210
+get_graph_buffer_ipc_meta(fptr_t _fa);
211
+void register_graph_buffers(fptr_t _fa,
212
+ const std::vector<std::vector<int64_t>>& handles,
213
const std::vector<std::vector<int64_t>>& offsets);
214
#endif
csrc/torch_bindings.cpp
+6
-15
@@ -411,27 +411,18 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cuda_utils), cuda_utils) {
411
TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
412
// Custom all-reduce kernels
413
custom_ar.def(
414
- "init_custom_ar(Tensor meta, Tensor rank_data, "
415
- "str[] handles, int[] offsets, int rank, "
416
- "bool full_nvlink) -> int");
414
+ "init_custom_ar(int[] ipc_tensors, Tensor rank_data, "
415
+ "int rank, bool full_nvlink) -> int");
416
custom_ar.impl("init_custom_ar", torch::kCUDA, &init_custom_ar);
418
-
419
- custom_ar.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()");
420
- custom_ar.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg);
421
-
417
custom_ar.def(
423
- "all_reduce_unreg(int fa, Tensor inp, Tensor reg_buffer, Tensor! out) -> "
424
- "()");
425
- custom_ar.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg);
418
+ "all_reduce(int fa, Tensor inp, Tensor! out, int reg_buffer, "
419
+ "int reg_buffer_sz_bytes) -> ()");
420
+ custom_ar.impl("all_reduce", torch::kCUDA, &all_reduce);
421
422
custom_ar.def("dispose", &dispose);
423
custom_ar.def("meta_size", &meta_size);
424
430
- custom_ar.def(
431
- "register_buffer(int fa, Tensor t, str[] handles, "
432
- "int[] offsets) -> ()");
433
- custom_ar.impl("register_buffer", torch::kCUDA, ®ister_buffer);
434
-
425
+ custom_ar.def("register_buffer", ®ister_buffer);
426
custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta);
427
custom_ar.def("register_graph_buffers", ®ister_graph_buffers);
428
}
tests/distributed/test_custom_all_reduce.py
+2
-2
@@ -95,13 +95,13 @@ def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
95
inp = torch.ones(sz, dtype=torch.float32, device=device)
96
out = inp
97
for _ in range(num_communication):
98
- out = fa.all_reduce_unreg(out)
98
+ out = fa.all_reduce(out, registered=False)
99
torch.testing.assert_close(out, inp * (tp_size**num_communication))
100
101
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
102
out = inp
103
for _ in range(num_communication):
104
- out = fa.all_reduce_unreg(out)
104
+ out = fa.all_reduce(out, registered=False)
105
torch.testing.assert_close(out, inp * (tp_size**num_communication))
106
107
tools/profiler/visualize_layerwise_profile.py
+16
-16
@@ -196,8 +196,8 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
196
def is_cross_device_reduce_2stage(op_name: str):
197
return "cross_device_reduce_2stage" in op_name
198
199
- def is_custom_ar_all_reduce_unreg(op_name: str):
200
- return "_C_custom_ar::all_reduce_unreg" in op_name
199
+ def is_custom_ar_all_reduce(op_name: str):
200
+ return "_C_custom_ar::all_reduce" in op_name
201
202
def is_reduce_kernel(op_name: str):
203
return "reduce_kernel" in op_name
@@ -246,9 +246,9 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
246
filter(lambda x: is_cross_device_reduce_2stage(x), ops))
247
ops = list(filter(lambda x: x not in cross_device_reduce_2stage_ops, ops))
248
249
- custom_ar_all_reduce_unreg_ops = list(
250
- filter(lambda x: is_custom_ar_all_reduce_unreg(x), ops))
251
- ops = list(filter(lambda x: x not in custom_ar_all_reduce_unreg_ops, ops))
249
+ custom_ar_all_reduce_ops = list(
250
+ filter(lambda x: is_custom_ar_all_reduce(x), ops))
251
+ ops = list(filter(lambda x: x not in custom_ar_all_reduce_ops, ops))
252
253
reduce_kernel_ops = list(filter(lambda x: is_reduce_kernel(x), ops))
254
ops = list(filter(lambda x: x not in reduce_kernel_ops, ops))
@@ -289,21 +289,21 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
289
if len(cross_device_reduce_2stage_ops):
290
trace_df['cross_device_reduce_2stage_ops'] = trace_df[
291
cross_device_reduce_2stage_ops].agg("sum", axis=1)
292
- if len(custom_ar_all_reduce_unreg_ops):
293
- trace_df['custom_ar_all_reduce_unreg_ops'] = trace_df[
294
- custom_ar_all_reduce_unreg_ops].agg("sum", axis=1)
292
+ if len(custom_ar_all_reduce_ops):
293
+ trace_df['custom_ar_all_reduce_ops'] = trace_df[
294
+ custom_ar_all_reduce_ops].agg("sum", axis=1)
295
if len(reduce_kernel_ops):
296
trace_df['reduce_kernel_ops'] = trace_df[reduce_kernel_ops].agg("sum",
297
axis=1)
298
299
- trace_df.drop(
300
- attention_ops + quant_ops + gemm_ops + rms_norm_ops + vocab_embed_ops +
301
- mem_ops + elementwise_ops + nccl_all_reduce_ops + nccl_gather_ops +
302
- nccl_broadcast_ops + nccl_other_ops + cross_device_reduce_1stage_ops +
303
- cross_device_reduce_2stage_ops + custom_ar_all_reduce_unreg_ops +
304
- reduce_kernel_ops,
305
- axis=1,
306
- inplace=True)
299
+ trace_df.drop(attention_ops + quant_ops + gemm_ops + rms_norm_ops +
300
+ vocab_embed_ops + mem_ops + elementwise_ops +
301
+ nccl_all_reduce_ops + nccl_gather_ops + nccl_broadcast_ops +
302
+ nccl_other_ops + cross_device_reduce_1stage_ops +
303
+ cross_device_reduce_2stage_ops + custom_ar_all_reduce_ops +
304
+ reduce_kernel_ops,
305
+ axis=1,
306
+ inplace=True)
307
return trace_df
308
309
vllm/_custom_ops.py
+12
-17
@@ -912,20 +912,16 @@ def get_max_shared_memory_per_block_device_attribute(device: int) -> int:
912
913
914
# custom ar
915
-def init_custom_ar(meta: torch.Tensor, rank_data: torch.Tensor,
916
- handles: List[str], offsets: List[int], rank: int,
917
- full_nvlink: bool) -> int:
918
- return torch.ops._C_custom_ar.init_custom_ar(meta, rank_data, handles,
919
- offsets, rank, full_nvlink)
915
+def init_custom_ar(ipc_tensors: List[torch.Tensor], rank_data: torch.Tensor,
916
+ rank: int, full_nvlink: bool) -> int:
917
+ return torch.ops._C_custom_ar.init_custom_ar(ipc_tensors, rank_data, rank,
918
+ full_nvlink)
919
920
922
-def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None:
923
- torch.ops._C_custom_ar.all_reduce_reg(fa, inp, out)
924
-
925
-
926
-def all_reduce_unreg(fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor,
927
- out: torch.Tensor) -> None:
928
- torch.ops._C_custom_ar.all_reduce_unreg(fa, inp, reg_buffer, out)
921
+def all_reduce(fa: int, inp: torch.Tensor, out: torch.Tensor, reg_buffer: int,
922
+ reg_buffer_sz_bytes: int) -> None:
923
+ torch.ops._C_custom_ar.all_reduce(fa, inp, out, reg_buffer,
924
+ reg_buffer_sz_bytes)
925
926
927
def dispose(fa: int) -> None:
@@ -936,16 +932,15 @@ def meta_size() -> int:
932
return torch.ops._C_custom_ar.meta_size()
933
934
939
-def register_buffer(fa: int, t: torch.Tensor, handles: List[str],
940
- offsets: List[int]) -> None:
941
- return torch.ops._C_custom_ar.register_buffer(fa, t, handles, offsets)
935
+def register_buffer(fa: int, ipc_tensors: List[int]) -> None:
936
+ return torch.ops._C_custom_ar.register_buffer(fa, ipc_tensors)
937
938
944
-def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[str], List[int]]:
939
+def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[int], List[int]]:
940
return torch.ops._C_custom_ar.get_graph_buffer_ipc_meta(fa)
941
942
948
-def register_graph_buffers(fa: int, handles: List[str],
943
+def register_graph_buffers(fa: int, handles: List[List[int]],
944
offsets: List[List[int]]) -> None:
945
torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets)
946
vllm/distributed/device_communicators/custom_all_reduce.py
+54
-86
@@ -1,6 +1,6 @@
1
import ctypes
2
from contextlib import contextmanager
3
-from typing import Any, List, Optional, Union
3
+from typing import List, Optional, Union
4
5
import torch
6
import torch.distributed as dist
@@ -147,18 +147,14 @@ class CustomAllreduce:
147
return
148
149
self.disabled = False
150
- # buffers memory are owned by this Python class and passed to C++
151
- # meta data composes of two parts: meta data for synchronization
152
- # (256 bytes) and a temporary buffer for storing intermediate
153
- # allreduce results.
154
- self.meta = torch.zeros(ops.meta_size() + max_size,
155
- dtype=torch.uint8,
156
- device=self.device)
150
+ # Buffers memory are owned by this Python class and passed to C++.
151
+ # Meta data composes of two parts: meta data for synchronization and a
152
+ # temporary buffer for storing intermediate allreduce results.
153
+ self.meta_ptrs = self.create_shared_buffer(ops.meta_size() + max_size,
154
+ group=group)
155
# This is a pre-registered IPC buffer. In eager mode, input tensors
156
# are first copied into this buffer before allreduce is performed
159
- self.buffer = torch.empty(max_size,
160
- dtype=torch.uint8,
161
- device=self.device)
157
+ self.buffer_ptrs = self.create_shared_buffer(max_size, group=group)
158
# This is a buffer for storing the tuples of pointers pointing to
159
# IPC buffers from all ranks. Each registered tuple has size of
160
# 8*world_size bytes where world_size is at most 8. Allocating 8MB
@@ -170,16 +166,19 @@ class CustomAllreduce:
166
self.max_size = max_size
167
self.rank = rank
168
self.world_size = world_size
173
- handles, offsets = self._get_ipc_meta(self.meta)
169
self.full_nvlink = full_nvlink
175
- self._ptr = ops.init_custom_ar(self.meta, self.rank_data, handles,
176
- offsets, rank, self.full_nvlink)
177
- self.register_buffer(self.buffer)
170
+ self._ptr = ops.init_custom_ar(self.meta_ptrs, self.rank_data, rank,
171
+ self.full_nvlink)
172
+ ops.register_buffer(self._ptr, self.buffer_ptrs)
173
174
@staticmethod
175
def create_shared_buffer(
176
size_in_bytes: int,
177
group: Optional[ProcessGroup] = None) -> List[int]:
178
+ """
179
+ Creates a shared buffer and returns a list of pointers
180
+ representing the buffer on all processes in the group.
181
+ """
182
lib = CudaRTLibrary()
183
pointer = lib.cudaMalloc(size_in_bytes)
184
handle = lib.cudaIpcGetMemHandle(pointer)
@@ -220,60 +219,24 @@ class CustomAllreduce:
219
if not self.disabled:
220
self.register_graph_buffers()
221
223
- def _get_ipc_meta(self, inp: torch.Tensor):
224
- data = inp.untyped_storage()._share_cuda_()
225
- handle = data[1]
226
- # https://github.com/pytorch/pytorch/pull/130890 changes
227
- # the binary format of the ipc handle
228
- # it starts from pytorch 2.5
229
- if len(handle) > 64:
230
- assert len(handle) == 66
231
- # only support SHAREABLE_HANDLE_VERSION = 1
232
- assert int(handle[0]) == 1
233
- # only support SHAREABLE_CUDA_MALLOC = 'c'
234
- assert handle[1] == ord("c")
235
- handle = handle[2:]
236
- # TODO: support expandable segment
237
- shard_data = (
238
- handle, # ipc handle to base ptr
239
- data[3], # offset of base ptr
240
- )
241
- return self._gather_ipc_meta(shard_data)
242
-
243
- def _gather_ipc_meta(self, shard_data):
244
- # Note: don't use `[[None]] * self.world_size` here
245
- # because it will create a list of the same reference
246
- all_data: List[Optional[Any]] = [[None]
247
- for i in range(self.world_size)]
248
- all_data[self.rank][0] = shard_data
249
-
250
- ranks = dist.get_process_group_ranks(group=self.group)
251
- ranks.sort()
222
+ def register_graph_buffers(self):
223
+ handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr)
224
+ logger.info("Registering %d cuda graph addresses", len(offset))
225
+ # We cannot directly use `dist.all_gather_object` here
226
+ # because it is incompatible with `gloo` backend under inference mode.
227
+ # see https://github.com/pytorch/pytorch/issues/126032 for details.
228
+ all_data = [[None, None]
229
+ for _ in range(dist.get_world_size(group=self.group))]
230
+ all_data[self.rank] = [handle, offset]
231
+ ranks = sorted(dist.get_process_group_ranks(group=self.group))
232
for i, rank in enumerate(ranks):
233
dist.broadcast_object_list(all_data[i],
234
src=rank,
235
group=self.group,
236
device="cpu")
257
-
258
- # we cannot directly use `dist.all_gather_object` here
259
- # because it is incompatible with `gloo` backend under inference mode.
260
- # see https://github.com/pytorch/pytorch/issues/126032 for details.
261
-
262
- handles = []
263
- offsets = []
264
- for i in range(len(all_data)):
265
- handles.append(all_data[i][0][0]) # type: ignore
266
- offsets.append(all_data[i][0][1]) # type: ignore
267
- return handles, offsets
268
-
269
- def register_buffer(self, inp: torch.Tensor):
270
- handles, offsets = self._get_ipc_meta(inp)
271
- ops.register_buffer(self._ptr, inp, handles, offsets)
272
-
273
- def register_graph_buffers(self):
274
- handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr)
275
- handles, offsets = self._gather_ipc_meta((bytes(handle), offset))
276
- logger.info("Registering %d cuda graph addresses", len(offset))
237
+ # Unpack list of tuples to tuple of lists.
238
+ handles = [d[0] for d in all_data] # type: ignore
239
+ offsets = [d[1] for d in all_data] # type: ignore
240
ops.register_graph_buffers(self._ptr, handles, offsets)
241
242
def should_custom_ar(self, inp: torch.Tensor):
@@ -291,45 +254,50 @@ class CustomAllreduce:
254
return inp_size < self.max_size
255
return False
256
294
- # all reduce, assuming inp tensor is IPC registered with register_buffer,
295
- # or, in the context of cuda graphs, register_graph_buffers
296
- def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None):
297
- if out is None:
298
- out = torch.empty_like(inp)
299
- ops.all_reduce_reg(self._ptr, inp, out)
300
- return out
301
-
302
- # all reduce, assuming inp tensor is NOT IPC registered
303
- def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None):
257
+ def all_reduce(self,
258
+ inp: torch.Tensor,
259
+ *,
260
+ out: torch.Tensor = None,
261
+ registered: bool = False):
262
+ """Performs an out-of-place all reduce.
263
+
264
+ If registered is True, this assumes inp's pointer is already
265
+ IPC-registered. Otherwise, inp is first copied into a pre-registered
266
+ buffer.
267
+ """
268
if out is None:
269
out = torch.empty_like(inp)
306
- ops.all_reduce_unreg(self._ptr, inp, self.buffer, out)
270
+ if registered:
271
+ ops.all_reduce(self._ptr, inp, out, 0, 0)
272
+ else:
273
+ ops.all_reduce(self._ptr, inp, out, self.buffer_ptrs[self.rank],
274
+ self.max_size)
275
return out
276
277
def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
310
- # when custom allreduce is disabled, this will be None
278
+ """The main allreduce API that provides support for cuda graph."""
279
+ # When custom allreduce is disabled, this will be None.
280
if self.disabled or not self.should_custom_ar(input):
281
return None
282
if self._IS_CAPTURING:
283
if torch.cuda.is_current_stream_capturing():
315
- return self.all_reduce_reg(input)
284
+ return self.all_reduce(input, registered=True)
285
else:
317
- # if warm up, mimic the allocation pattern
318
- # since custom allreduce is out-of-place
286
+ # If warm up, mimic the allocation pattern since custom
287
+ # allreduce is out-of-place.
288
return torch.empty_like(input)
289
else:
321
- # note: outside of cuda graph context,
322
- # custom allreduce incurs a cost of cudaMemcpy, which should
323
- # be small(<=1% of overall latency) compared to the performance
324
- # gains of using custom kernels
325
- return self.all_reduce_unreg(input)
326
-
327
- return None
290
+ # Note: outside of cuda graph context, custom allreduce incurs a
291
+ # cost of cudaMemcpy, which should be small (<=1% of overall
292
+ # latency) compared to the performance gain of using custom kernels
293
+ return self.all_reduce(input, registered=False)
294
295
def close(self):
296
if not self.disabled and self._ptr:
297
ops.dispose(self._ptr)
298
self._ptr = 0
299
+ self.free_shared_buffer(self.meta_ptrs)
300
+ self.free_shared_buffer(self.buffer_ptrs)
301
302
def __del__(self):
303
self.close()