A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Implement custom all reduce kernels (#2192)
Hanzhi Zhou committed
Jan 28, 2024 at 04:46 UTC
380170038e05cf81953c29d7e8ed789e048b6434
18 files changed
+1453
-65
csrc/custom_all_reduce.cu
new
+148
@@ -0,0 +1,148 @@
1
+#include <ATen/cuda/Exceptions.h>
2
+#include <c10/cuda/CUDAGuard.h>
3
+#include <c10/cuda/CUDAStream.h>
4
+#include <torch/extension.h>
5
+
6
+#include "custom_all_reduce.cuh"
7
+
8
+// fake pointer type
9
+using fptr_t = uint64_t;
10
+static_assert(sizeof(void *) == sizeof(fptr_t));
11
+
12
+fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
13
+ const std::vector<std::string> &handles,
14
+ const std::vector<int64_t> &offsets, int rank,
15
+ bool full_nvlink) {
16
+ int world_size = offsets.size();
17
+ if (world_size > 8)
18
+ throw std::invalid_argument("world size > 8 is not supported");
19
+ if (world_size % 2 != 0)
20
+ throw std::invalid_argument("Odd num gpus is not supported for now");
21
+ if (world_size != handles.size())
22
+ throw std::invalid_argument(
23
+ "handles length should equal to offsets length");
24
+ if (rank < 0 || rank >= world_size)
25
+ throw std::invalid_argument("invalid rank passed in");
26
+
27
+ cudaIpcMemHandle_t ipc_handles[8];
28
+ for (int i = 0; i < world_size; i++) {
29
+ std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
30
+ }
31
+ return (fptr_t) new vllm::CustomAllreduce(
32
+ reinterpret_cast<vllm::Metadata *>(meta.data_ptr()), rank_data.data_ptr(),
33
+ rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
34
+}
35
+
36
+/**
37
+ * Make sure tensor t's data lies completely within ((char)t.data_ptr()) +
38
+ * t.numel() * t.element_size(). This is slightly weaker than t.is_contiguous()
39
+ * because it allows transpose of contiguous slice (i.e. slicing the first
40
+ * dimension). Currently, we require this because stride information is not
41
+ * passed into the kernels and we treat input tensors as flat.
42
+ *
43
+ * Examples
44
+ * A = torch.zeros(3, 3, 3)
45
+ * 1. A: OK
46
+ * 2. A[1:]: OK
47
+ * 3. A.permute(2, 0, 1): OK
48
+ * 4. A[1:].permute(2, 0, 1): OK
49
+ * 5. A[None].expand(2, -1, -1, -1): Not OK
50
+ * 6. A[:, 1:, 1:]: Not OK
51
+ */
52
+bool _is_weak_contiguous(torch::Tensor &t) {
53
+ return t.is_contiguous() ||
54
+ (t.storage().nbytes() - t.storage_offset() * t.element_size() ==
55
+ t.numel() * t.element_size());
56
+}
57
+
58
+bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
59
+ bool full_nvlink) {
60
+ auto inp_size = inp.numel() * inp.element_size();
61
+ // custom allreduce requires input byte size to be multiples of 16
62
+ if (inp_size % 16 != 0) return false;
63
+ if (!_is_weak_contiguous(inp)) return false;
64
+ if (world_size == 2 || full_nvlink) return inp_size <= max_size;
65
+ // 4 PCIE GPUs use 2 stage allreduce, and is only faster than NCCL when size
66
+ // <= 512k
67
+ return world_size <= 4 && inp_size <= 512 * 1024;
68
+}
69
+
70
+void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
71
+ cudaStream_t stream) {
72
+ auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
73
+ TORCH_CHECK(_is_weak_contiguous(out));
74
+ switch (out.scalar_type()) {
75
+ case at::ScalarType::Float: {
76
+ fa->allreduce<float>(stream, reinterpret_cast<float *>(inp.data_ptr()),
77
+ reinterpret_cast<float *>(out.data_ptr()),
78
+ out.numel());
79
+ break;
80
+ }
81
+ case at::ScalarType::Half: {
82
+ fa->allreduce<half>(stream, reinterpret_cast<half *>(inp.data_ptr()),
83
+ reinterpret_cast<half *>(out.data_ptr()),
84
+ out.numel());
85
+ break;
86
+ }
87
+#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
88
+ case at::ScalarType::BFloat16: {
89
+ fa->allreduce<nv_bfloat16>(
90
+ stream, reinterpret_cast<nv_bfloat16 *>(inp.data_ptr()),
91
+ reinterpret_cast<nv_bfloat16 *>(out.data_ptr()), out.numel());
92
+ break;
93
+ }
94
+#endif
95
+ default:
96
+ throw std::runtime_error(
97
+ "custom allreduce only supports float32, float16 and bfloat16");
98
+ }
99
+}
100
+
101
+void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out) {
102
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
103
+ auto stream = c10::cuda::getCurrentCUDAStream().stream();
104
+ TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
105
+ TORCH_CHECK_EQ(inp.numel(), out.numel());
106
+ _all_reduce(_fa, inp, out, stream);
107
+}
108
+
109
+void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor ®_buffer,
110
+ torch::Tensor &out) {
111
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
112
+ auto stream = c10::cuda::getCurrentCUDAStream().stream();
113
+
114
+ auto input_size = inp.numel() * inp.element_size();
115
+ TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
116
+ TORCH_CHECK_EQ(inp.numel(), out.numel());
117
+ TORCH_CHECK(input_size <= reg_buffer.numel() * reg_buffer.element_size(),
118
+ "registered buffer is too small to contain the input");
119
+ AT_CUDA_CHECK(cudaMemcpyAsync(reg_buffer.data_ptr(), inp.data_ptr(),
120
+ input_size, cudaMemcpyDeviceToDevice, stream));
121
+ _all_reduce(_fa, reg_buffer, out, stream);
122
+}
123
+
124
+void dispose(fptr_t _fa) {
125
+ auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
126
+ delete fa;
127
+}
128
+
129
+int meta_size() { return sizeof(vllm::Metadata); }
130
+
131
+void register_buffer(fptr_t _fa, torch::Tensor &t,
132
+ const std::vector<std::string> &handles,
133
+ const std::vector<int64_t> &offsets) {
134
+ auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
135
+ fa->register_buffer(handles, offsets, t.data_ptr());
136
+}
137
+
138
+std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
139
+ fptr_t _fa) {
140
+ auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
141
+ return fa->get_graph_buffer_ipc_meta();
142
+}
143
+
144
+void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
145
+ const std::vector<std::vector<int64_t>> &offsets) {
146
+ auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
147
+ fa->register_graph_buffers(handles, offsets);
148
+}
csrc/custom_all_reduce.cuh
new
+555
@@ -0,0 +1,555 @@
1
+#pragma once
2
+
3
+#include <cuda.h>
4
+#include <cuda_bf16.h>
5
+#include <cuda_fp16.h>
6
+#include <cuda_runtime.h>
7
+
8
+#include <iostream>
9
+#include <limits>
10
+#include <unordered_map>
11
+#include <vector>
12
+
13
+#define CUDACHECK(cmd) \
14
+ do { \
15
+ cudaError_t e = cmd; \
16
+ if (e != cudaSuccess) { \
17
+ printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \
18
+ cudaGetErrorString(e)); \
19
+ exit(EXIT_FAILURE); \
20
+ } \
21
+ } while (0)
22
+
23
+namespace vllm {
24
+
25
+struct Signal {
26
+ alignas(64) union {
27
+ uint64_t flag;
28
+ unsigned char data[8];
29
+ } start;
30
+ alignas(64) union {
31
+ uint64_t flag;
32
+ unsigned char data[8];
33
+ } end;
34
+};
35
+
36
+struct Metadata {
37
+ alignas(128) Signal sg;
38
+ alignas(128) int counter;
39
+};
40
+static_assert(offsetof(Metadata, counter) == 128);
41
+static_assert(sizeof(Metadata) == 256);
42
+
43
+struct __align__(16) RankData { const void *__restrict__ ptrs[8]; };
44
+
45
+struct RankSignals {
46
+ volatile Signal *signals[8];
47
+};
48
+
49
+// like std::array, but aligned
50
+template <typename T, int sz>
51
+struct __align__(alignof(T) * sz) array_t {
52
+ T data[sz];
53
+ using type = T;
54
+ static constexpr int size = sz;
55
+};
56
+
57
+// use packed type to maximize memory efficiency
58
+// goal: generate ld.128 and st.128 instructions
59
+template <typename T>
60
+struct packed_t {
61
+ // the (P)acked type for load/store
62
+ using P = array_t<T, 16 / sizeof(T)>;
63
+ // the (A)ccumulator type for reduction
64
+ using A = array_t<float, 16 / sizeof(T)>;
65
+};
66
+
67
+#define DINLINE __device__ __forceinline__
68
+
69
+// scalar cast functions
70
+DINLINE float upcast_s(half val) { return __half2float(val); }
71
+
72
+template <typename T>
73
+DINLINE T downcast_s(float val);
74
+template <>
75
+DINLINE half downcast_s(float val) {
76
+ return __float2half(val);
77
+}
78
+
79
+// scalar add functions
80
+// for some reason when compiling with Pytorch, the + operator for half and
81
+// bfloat is disabled so we call the intrinsics directly
82
+DINLINE half &assign_add(half &a, half b) {
83
+ a = __hadd(a, b);
84
+ return a;
85
+}
86
+DINLINE float &assign_add(float &a, float b) { return a += b; }
87
+
88
+#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
89
+DINLINE float upcast_s(nv_bfloat16 val) { return __bfloat162float(val); }
90
+template <>
91
+DINLINE nv_bfloat16 downcast_s(float val) {
92
+ return __float2bfloat16(val);
93
+}
94
+DINLINE nv_bfloat16 &assign_add(nv_bfloat16 &a, nv_bfloat16 b) {
95
+ a = __hadd(a, b);
96
+ return a;
97
+}
98
+#endif
99
+
100
+template <typename T, int N>
101
+DINLINE array_t<T, N> &packed_assign_add(array_t<T, N> &a, array_t<T, N> b) {
102
+#pragma unroll
103
+ for (int i = 0; i < N; i++) {
104
+ assign_add(a.data[i], b.data[i]);
105
+ }
106
+ return a;
107
+}
108
+
109
+template <typename T, int N>
110
+DINLINE array_t<float, N> upcast(array_t<T, N> val) {
111
+ if constexpr (std::is_same<T, float>::value) {
112
+ return val;
113
+ } else {
114
+ array_t<float, N> out;
115
+#pragma unroll
116
+ for (int i = 0; i < N; i++) {
117
+ out.data[i] = upcast_s(val.data[i]);
118
+ }
119
+ return out;
120
+ }
121
+}
122
+
123
+template <typename O>
124
+DINLINE O downcast(array_t<float, O::size> val) {
125
+ if constexpr (std::is_same<typename O::type, float>::value) {
126
+ return val;
127
+ } else {
128
+ O out;
129
+#pragma unroll
130
+ for (int i = 0; i < O::size; i++) {
131
+ out.data[i] = downcast_s<typename O::type>(val.data[i]);
132
+ }
133
+ return out;
134
+ }
135
+}
136
+
137
+// compute flag at compile time
138
+__host__ __device__ constexpr uint64_t compute_flag(int ngpus) {
139
+ auto m = std::numeric_limits<uint64_t>::max();
140
+ return m >> ((8 - ngpus) * 8);
141
+}
142
+
143
+template <int ngpus>
144
+DINLINE void start_sync(const RankSignals &sg, volatile Metadata *meta,
145
+ int rank) {
146
+ constexpr auto FLAG = compute_flag(ngpus);
147
+ if (blockIdx.x == 0) {
148
+ if (threadIdx.x < ngpus)
149
+ // simultaneously write to the corresponding byte to all other ranks.
150
+ // Latency = 1 p2p write
151
+ sg.signals[threadIdx.x]->start.data[rank] = 255;
152
+ else if (threadIdx.x == 32)
153
+ // reset
154
+ meta->sg.end.flag = 0;
155
+ }
156
+ if (threadIdx.x == 0) {
157
+ while (meta->sg.start.flag != FLAG)
158
+ ;
159
+ }
160
+ __syncthreads();
161
+}
162
+
163
+template <int ngpus, bool final_sync = false>
164
+DINLINE void end_sync(const RankSignals &sg, volatile Metadata *meta,
165
+ int rank) {
166
+ constexpr auto FLAG = compute_flag(ngpus);
167
+ __syncthreads();
168
+ __shared__ int num;
169
+ if (threadIdx.x == 0) num = atomicAdd((int *)&meta->counter, 1);
170
+ __syncthreads();
171
+
172
+ // Only the last completing block can perform the end synchronization
173
+ // This can ensures when the final busy wait ends, all ranks must have
174
+ // finished reading each other's buffer.
175
+ if (num == gridDim.x - 1) {
176
+ if (threadIdx.x == 32) {
177
+ // reset in a different warp
178
+ meta->counter = 0;
179
+ meta->sg.start.flag = 0;
180
+ } else if (threadIdx.x < ngpus) {
181
+ // simultaneously write to the corresponding byte to all other ranks.
182
+ // Latency = 1 p2p write
183
+ sg.signals[threadIdx.x]->end.data[rank] = 255;
184
+ }
185
+ // if this is the final sync, only one block needs it
186
+ // because kernel exit can serve as sync
187
+ if constexpr (final_sync) {
188
+ if (threadIdx.x == 0) {
189
+ while (meta->sg.end.flag != FLAG)
190
+ ;
191
+ }
192
+ }
193
+ }
194
+ if constexpr (!final_sync) {
195
+ if (threadIdx.x == 0) {
196
+ while (meta->sg.end.flag != FLAG)
197
+ ;
198
+ }
199
+ __syncthreads();
200
+ }
201
+}
202
+
203
+template <typename P, int ngpus, typename A>
204
+DINLINE P packed_reduce(const P *ptrs[], int idx) {
205
+ A tmp = upcast(ptrs[0][idx]);
206
+#pragma unroll
207
+ for (int i = 1; i < ngpus; i++) {
208
+ packed_assign_add(tmp, upcast(ptrs[i][idx]));
209
+ }
210
+ return downcast<P>(tmp);
211
+}
212
+
213
+template <typename T, int ngpus>
214
+__global__ void __launch_bounds__(512, 1)
215
+ cross_device_reduce_1stage(RankData *_dp, RankSignals sg,
216
+ volatile Metadata *meta, T *__restrict__ result,
217
+ int rank, int size) {
218
+ using P = typename packed_t<T>::P;
219
+ using A = typename packed_t<T>::A;
220
+ // note: we don't reorder the address so the accumulation order is the same
221
+ // for all ranks, ensuring bitwise identical results
222
+ auto dp = *_dp;
223
+ start_sync<ngpus>(sg, meta, rank);
224
+ // do the actual reduction
225
+ for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
226
+ idx += gridDim.x * blockDim.x) {
227
+ ((P *)result)[idx] =
228
+ packed_reduce<P, ngpus, A>((const P **)&dp.ptrs[0], idx);
229
+ }
230
+ end_sync<ngpus, true>(sg, meta, rank);
231
+}
232
+
233
+template <typename P>
234
+DINLINE P *get_tmp_buf(volatile Signal *sg) {
235
+ return (P *)(((Metadata *)sg) + 1);
236
+}
237
+
238
+template <typename T, int ngpus>
239
+__global__ void __launch_bounds__(512, 1)
240
+ cross_device_reduce_2stage(RankData *_dp, RankSignals sg,
241
+ volatile Metadata *meta, T *__restrict__ result,
242
+ int rank, int size) {
243
+ int tid = blockIdx.x * blockDim.x + threadIdx.x;
244
+ int stride = gridDim.x * blockDim.x;
245
+ using P = typename packed_t<T>::P;
246
+ using A = typename packed_t<T>::A;
247
+ int part = size / ngpus;
248
+ int start = rank * part;
249
+ int end = rank == ngpus - 1 ? size : start + part;
250
+ const P *ptrs[ngpus];
251
+ P *tmps[ngpus];
252
+#pragma unroll
253
+ for (int i = 0; i < ngpus; i++) {
254
+ int target = (rank + i) % ngpus;
255
+ ptrs[i] = (const P *)_dp->ptrs[target];
256
+ tmps[i] = get_tmp_buf<P>(sg.signals[target]);
257
+ }
258
+ auto tmp_out = tmps[0];
259
+ start_sync<ngpus>(sg, meta, rank);
260
+ // stage 1: reduce scatter
261
+ for (int idx = start + tid; idx < end; idx += stride) {
262
+ tmp_out[idx - start] = packed_reduce<P, ngpus, A>(ptrs, idx);
263
+ }
264
+ // Maybe TODO: replace this with per-block release-acquire
265
+ // can save about 1-2us (not a lot though)
266
+ end_sync<ngpus>(sg, meta, rank);
267
+
268
+ // stage 2: allgather
269
+ for (int idx = tid; idx < part; idx += stride) {
270
+#pragma unroll
271
+ for (int i = 0; i < ngpus; i++) {
272
+ int dst_idx = ((rank + i) % ngpus) * part + idx;
273
+ ((P *)result)[dst_idx] = tmps[i][idx];
274
+ }
275
+ }
276
+ // process the last larger partition
277
+ int remaining = size - part * ngpus;
278
+ if (tid < remaining) {
279
+ int dst_idx = tid + part * ngpus;
280
+ ((P *)result)[dst_idx] = get_tmp_buf<P>(sg.signals[ngpus - 1])[part + tid];
281
+ }
282
+
283
+ // faster than this
284
+ // for (int idx = tid; idx < size; idx += stride) {
285
+ // int target_rank = idx / part;
286
+ // if (target_rank == ngpus) target_rank -= 1;
287
+ // ((P *)result)[idx] = tmps[target_rank][idx - target_rank * part];
288
+ // }
289
+}
290
+
291
+template <typename T, int ngpus>
292
+__global__ void __launch_bounds__(512, 1)
293
+ cross_device_reduce_half_butterfly(RankData *_dp, RankSignals sg,
294
+ volatile Metadata *meta,
295
+ T *__restrict__ result, int rank,
296
+ int size) {
297
+ int tid = blockIdx.x * blockDim.x + threadIdx.x;
298
+ int stride = gridDim.x * blockDim.x;
299
+ using P = typename packed_t<T>::P;
300
+ using A = typename packed_t<T>::A;
301
+ auto tmp_out = get_tmp_buf<P>(sg.signals[rank]);
302
+ constexpr int hg = ngpus / 2;
303
+ // Actually not quite half butterfly.
304
+ // This is an all-to-all within each group containing half of the ranks
305
+ // followed by cross-group add. Equivalent to half butterfly when there
306
+ // are 4 GPUs, a common case for PCIe cards like T4 and A10.
307
+ const P *ptrs[hg];
308
+ {
309
+ int start = rank - rank % hg;
310
+#pragma unroll
311
+ for (int i = 0; i < hg; i++) {
312
+ ptrs[i] = (const P *)_dp->ptrs[i + start];
313
+ }
314
+ }
315
+ start_sync<ngpus>(sg, meta, rank);
316
+ for (int idx = tid; idx < size; idx += stride) {
317
+ tmp_out[idx] = packed_reduce<P, hg, A>(ptrs, idx);
318
+ }
319
+ end_sync<ngpus>(sg, meta, rank);
320
+
321
+ auto src = get_tmp_buf<P>(sg.signals[(ngpus - 1) - rank % ngpus]);
322
+ // do the cross group reduction
323
+ for (int idx = tid; idx < size; idx += stride) {
324
+ auto tmp = tmp_out[idx];
325
+ packed_assign_add(tmp, src[idx]);
326
+ ((P *)result)[idx] = tmp;
327
+ }
328
+}
329
+
330
+class CustomAllreduce {
331
+ public:
332
+ int rank_;
333
+ int world_size_;
334
+ bool full_nvlink_;
335
+
336
+ // below are device pointers
337
+ RankSignals sg_;
338
+ std::unordered_map<void *, RankData *> buffers_;
339
+ Metadata *meta_;
340
+
341
+ // stores the registered device pointers from all ranks
342
+ RankData *d_rank_data_base_, *d_rank_data_end_;
343
+ std::vector<void *> graph_unreg_buffers_;
344
+ std::vector<void *> ipc_handles_;
345
+
346
+ /**
347
+ * meta is a pointer to device metadata and temporary buffer for allreduce.
348
+ *
349
+ * There's a total of sizeof(Metadata) of prefix before the actual data,
350
+ * so meta + 1 points to actual temporary buffer.
351
+ *
352
+ * note: this class does not own any device memory. Any required buffers
353
+ * are passed in from the constructor
354
+ */
355
+ CustomAllreduce(Metadata *meta, void *rank_data, size_t rank_data_sz,
356
+ const cudaIpcMemHandle_t *handles,
357
+ const std::vector<int64_t> &offsets, int rank,
358
+ bool full_nvlink = true)
359
+ : rank_(rank),
360
+ world_size_(offsets.size()),
361
+ full_nvlink_(full_nvlink),
362
+ meta_(meta),
363
+ d_rank_data_base_(reinterpret_cast<RankData *>(rank_data)),
364
+ d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
365
+ for (int i = 0; i < world_size_; i++) {
366
+ Metadata *rank_meta;
367
+ if (i != rank_) {
368
+ char *handle;
369
+ CUDACHECK(cudaIpcOpenMemHandle((void **)&handle, handles[i],
370
+ cudaIpcMemLazyEnablePeerAccess));
371
+ ipc_handles_.push_back(handle);
372
+ handle += offsets[i];
373
+ rank_meta = (Metadata *)handle;
374
+ } else {
375
+ rank_meta = meta_;
376
+ }
377
+ sg_.signals[i] = &rank_meta->sg;
378
+ }
379
+ }
380
+
381
+ std::pair<std::vector<uint8_t>, std::vector<int64_t>>
382
+ get_graph_buffer_ipc_meta() {
383
+ auto num_buffers = graph_unreg_buffers_.size();
384
+ auto handle_sz = sizeof(cudaIpcMemHandle_t);
385
+ std::vector<uint8_t> handles(handle_sz * num_buffers, 0);
386
+ std::vector<int64_t> offsets(num_buffers);
387
+ for (int i = 0; i < num_buffers; i++) {
388
+ auto ptr = graph_unreg_buffers_[i];
389
+ void *base_ptr;
390
+ // note: must share the base address of each allocation, or we get wrong
391
+ // address
392
+ if (cuPointerGetAttribute(&base_ptr,
393
+ CU_POINTER_ATTRIBUTE_RANGE_START_ADDR,
394
+ (CUdeviceptr)ptr) != CUDA_SUCCESS)
395
+ throw std::runtime_error("failed to get pointer attr");
396
+ CUDACHECK(cudaIpcGetMemHandle(
397
+ (cudaIpcMemHandle_t *)&handles[i * handle_sz], base_ptr));
398
+ offsets[i] = ((char *)ptr) - ((char *)base_ptr);
399
+ }
400
+ return std::make_pair(handles, offsets);
401
+ }
402
+
403
+ void check_rank_data_capacity(size_t num = 1) {
404
+ if (d_rank_data_base_ + num > d_rank_data_end_)
405
+ throw std::runtime_error(
406
+ "Rank data buffer is overflowed by " +
407
+ std::to_string(d_rank_data_base_ + num - d_rank_data_end_));
408
+ }
409
+
410
+ void register_buffer(const std::vector<std::string> &handles,
411
+ const std::vector<int64_t> &offsets, void *self) {
412
+ check_rank_data_capacity();
413
+ RankData data;
414
+ for (int i = 0; i < world_size_; i++) {
415
+ if (i != rank_) {
416
+ char *handle;
417
+ CUDACHECK(cudaIpcOpenMemHandle(
418
+ (void **)&handle, *((const cudaIpcMemHandle_t *)handles[i].data()),
419
+ cudaIpcMemLazyEnablePeerAccess));
420
+ ipc_handles_.push_back(handle);
421
+ handle += offsets[i];
422
+ data.ptrs[i] = handle;
423
+ } else {
424
+ data.ptrs[i] = self;
425
+ }
426
+ }
427
+ auto d_data = d_rank_data_base_++;
428
+ CUDACHECK(
429
+ cudaMemcpy(d_data, &data, sizeof(RankData), cudaMemcpyHostToDevice));
430
+ buffers_[self] = d_data;
431
+ }
432
+
433
+ // note: when registering graph buffers, we intentionally choose to not
434
+ // deduplicate the addresses. That means if the allocator reuses some
435
+ // addresses, they will be registered again. This is to account for the remote
436
+ // possibility of different allocation patterns between ranks. For example,
437
+ // rank 1 may get the same input address for the second allreduce, but rank 2
438
+ // got a different address. IPC handles have internal reference counting
439
+ // mechanism so overhead should be small.
440
+ void register_graph_buffers(
441
+ const std::vector<std::string> &handles,
442
+ const std::vector<std::vector<int64_t>> &offsets) {
443
+ auto num_buffers = graph_unreg_buffers_.size();
444
+ check_rank_data_capacity(num_buffers);
445
+ std::vector<RankData> rank_data(num_buffers);
446
+ for (int i = 0; i < num_buffers; i++) {
447
+ auto self_ptr = graph_unreg_buffers_[i];
448
+ auto &rd = rank_data[i];
449
+ for (int j = 0; j < world_size_; j++) {
450
+ if (j != rank_) {
451
+ char *handle;
452
+ CUDACHECK(cudaIpcOpenMemHandle(
453
+ (void **)&handle,
454
+ *((cudaIpcMemHandle_t *)&handles[j]
455
+ [i * sizeof(cudaIpcMemHandle_t)]),
456
+ cudaIpcMemLazyEnablePeerAccess));
457
+ ipc_handles_.push_back(handle);
458
+ handle += offsets[j][i];
459
+ rd.ptrs[j] = handle;
460
+ } else {
461
+ rd.ptrs[j] = self_ptr;
462
+ }
463
+ }
464
+ }
465
+ CUDACHECK(cudaMemcpy(d_rank_data_base_, rank_data.data(),
466
+ sizeof(RankData) * num_buffers,
467
+ cudaMemcpyHostToDevice));
468
+ d_rank_data_base_ += num_buffers;
469
+ graph_unreg_buffers_.clear();
470
+ }
471
+
472
+ /**
473
+ * This is the result after careful grid search. Using 36 blocks give the best
474
+ * or close to the best runtime on the devices I tried: A100, A10, A30, T4,
475
+ * V100. You'll notice that NCCL kernels also only take a small amount of SMs.
476
+ * Not quite sure the underlying reason, but my guess is that too many SMs
477
+ * will cause contention on NVLink bus.
478
+ */
479
+ template <typename T>
480
+ void allreduce(cudaStream_t stream, T *input, T *output, int size,
481
+ int threads = 512, int block_limit = 36) {
482
+ auto d = packed_t<T>::P::size;
483
+ if (size % d != 0)
484
+ throw std::runtime_error(
485
+ "custom allreduce currently requires input length to be multiple "
486
+ "of " +
487
+ std::to_string(d));
488
+
489
+ RankData *ptrs;
490
+ cudaStreamCaptureStatus status;
491
+ CUDACHECK(cudaStreamIsCapturing(stream, &status));
492
+ if (status == cudaStreamCaptureStatusActive) {
493
+ ptrs = d_rank_data_base_ + graph_unreg_buffers_.size();
494
+ graph_unreg_buffers_.push_back(input);
495
+ } else {
496
+ auto it = buffers_.find(input);
497
+ if (it == buffers_.end())
498
+ throw std::runtime_error(
499
+ "buffer address " +
500
+ std::to_string(reinterpret_cast<uint64_t>(input)) +
501
+ " is not registered!");
502
+ ptrs = it->second;
503
+ }
504
+
505
+ size /= d;
506
+ auto bytes = size * sizeof(typename packed_t<T>::P);
507
+ int blocks = std::min(block_limit, (size + threads - 1) / threads);
508
+#define KL(ngpus, name) \
509
+ name<T, ngpus> \
510
+ <<<blocks, threads, 0, stream>>>(ptrs, sg_, meta_, output, rank_, size);
511
+#define REDUCE_CASE(ngpus) \
512
+ case ngpus: { \
513
+ if (world_size_ == 2) { \
514
+ KL(ngpus, cross_device_reduce_1stage); \
515
+ } else if (full_nvlink_) { \
516
+ if ((world_size_ <= 4 && bytes < 512 * 1024) || \
517
+ (world_size_ <= 8 && bytes < 256 * 1024)) { \
518
+ KL(ngpus, cross_device_reduce_1stage); \
519
+ } else { \
520
+ KL(ngpus, cross_device_reduce_2stage); \
521
+ } \
522
+ } else { \
523
+ KL(ngpus, cross_device_reduce_half_butterfly); \
524
+ } \
525
+ break; \
526
+ }
527
+
528
+ switch (world_size_) {
529
+ REDUCE_CASE(2)
530
+ REDUCE_CASE(4)
531
+ REDUCE_CASE(6)
532
+ REDUCE_CASE(8)
533
+ default:
534
+ throw std::runtime_error(
535
+ "custom allreduce only supports num gpus in (2,4,6,8). Actual num "
536
+ "gpus = " +
537
+ std::to_string(world_size_));
538
+ }
539
+#undef REDUCE_CASE
540
+#undef KL
541
+ }
542
+
543
+ ~CustomAllreduce() {
544
+ for (auto ptr : ipc_handles_) {
545
+ CUDACHECK(cudaIpcCloseMemHandle(ptr));
546
+ }
547
+ }
548
+};
549
+/**
550
+ * To inspect PTX/SASS, copy paste this header file to compiler explorer and add
551
+ a template instantiation:
552
+ * template void CustomAllreduce::allreduce<half>(cudaStream_t, half *, half *,
553
+ int, int, int);
554
+*/
555
+} // namespace vllm
csrc/custom_all_reduce_test.cu
new
+284
@@ -0,0 +1,284 @@
1
+/**
2
+ * This is a standalone test for custom allreduce.
3
+ * To compile, make sure you have MPI and NCCL installed in your system.
4
+ * export MPI_HOME=XXX
5
+ * nvcc -O2 -arch=native -std=c++17 custom_all_reduce_test.cu -o
6
+ * custom_all_reduce_test -lnccl -I${MPI_HOME}/include -lmpi
7
+ *
8
+ * Warning: this C++ test is not designed to be very readable and was used
9
+ * during the rapid prototyping process.
10
+ *
11
+ * To run:
12
+ * mpirun -np 8 ./custom_all_reduce_test
13
+ */
14
+#include <cuda.h>
15
+#include <curand_kernel.h>
16
+#include <stdio.h>
17
+#include <stdlib.h>
18
+
19
+#include <limits>
20
+#include <vector>
21
+
22
+#include "cuda_profiler_api.h"
23
+#include "custom_all_reduce.cuh"
24
+#include "mpi.h"
25
+#include "nccl.h"
26
+
27
+#define MPICHECK(cmd) \
28
+ do { \
29
+ int e = cmd; \
30
+ if (e != MPI_SUCCESS) { \
31
+ printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \
32
+ exit(EXIT_FAILURE); \
33
+ } \
34
+ } while (0)
35
+
36
+#define NCCLCHECK(cmd) \
37
+ do { \
38
+ ncclResult_t r = cmd; \
39
+ if (r != ncclSuccess) { \
40
+ printf("Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \
41
+ ncclGetErrorString(r)); \
42
+ exit(EXIT_FAILURE); \
43
+ } \
44
+ } while (0)
45
+
46
+__global__ void dummy_kernel() {
47
+ for (int i = 0; i < 100; i++) __nanosleep(1000000); // 100ms
48
+}
49
+
50
+template <typename T>
51
+__global__ void set_data(T *data, int size, int myRank) {
52
+ for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
53
+ idx += gridDim.x * blockDim.x) {
54
+ data[idx] = myRank * 0.11f;
55
+ }
56
+}
57
+
58
+template <typename T>
59
+__global__ void convert_data(const T *data1, const T *data2, double *fdata1,
60
+ double *fdata2, int size) {
61
+ for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
62
+ idx += gridDim.x * blockDim.x) {
63
+ fdata1[idx] = data1[idx];
64
+ fdata2[idx] = data2[idx];
65
+ }
66
+}
67
+
68
+__global__ void init_rand(curandState_t *state, int size, int nRanks) {
69
+ for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
70
+ idx += gridDim.x * blockDim.x) {
71
+ for (int i = 0; i < nRanks; i++) {
72
+ curand_init(i + 1, idx, 0, &state[idx * nRanks + i]);
73
+ }
74
+ }
75
+}
76
+
77
+template <typename T>
78
+__global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
79
+ int myRank, int nRanks, int size) {
80
+ for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
81
+ idx += gridDim.x * blockDim.x) {
82
+ double sum = 0.0;
83
+ for (int i = 0; i < nRanks; i++) {
84
+ double val = curand_uniform_double(&state[idx * nRanks + i]) * 4;
85
+ T hval = val; // downcast first
86
+ sum += static_cast<double>(hval);
87
+ if (i == myRank) data[idx] = hval;
88
+ }
89
+ ground_truth[idx] = sum;
90
+ }
91
+}
92
+
93
+template <typename T>
94
+void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
95
+ int data_size) {
96
+ T *result;
97
+ cudaStream_t stream;
98
+ CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
99
+ CUDACHECK(cudaMalloc(&result, data_size * sizeof(T)));
100
+ CUDACHECK(cudaMemset(result, 0, data_size * sizeof(T)));
101
+
102
+ cudaIpcMemHandle_t self_data_handle;
103
+ cudaIpcMemHandle_t data_handles[8];
104
+ vllm::Metadata *buffer;
105
+ T *self_data_copy;
106
+ /**
107
+ * Allocate IPC buffer
108
+ *
109
+ * The first section is a temporary buffer for storing intermediate allreduce
110
+ * results, if a particular algorithm requires it. The second section is for
111
+ * the input to the allreduce. The actual API takes the input pointer as an
112
+ * argument (that is, they can and usually should be allocated separately).
113
+ * But since the input pointers and the temporary buffer all require IPC
114
+ * registration, they are allocated and registered together in the test for
115
+ * convenience.
116
+ */
117
+ CUDACHECK(
118
+ cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
119
+ CUDACHECK(cudaMemset(buffer, 0,
120
+ 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
121
+ CUDACHECK(cudaMalloc(&self_data_copy, data_size * sizeof(T)));
122
+ CUDACHECK(cudaIpcGetMemHandle(&self_data_handle, buffer));
123
+
124
+ MPICHECK(MPI_Allgather(&self_data_handle, sizeof(cudaIpcMemHandle_t),
125
+ MPI_BYTE, data_handles, sizeof(cudaIpcMemHandle_t),
126
+ MPI_BYTE, MPI_COMM_WORLD));
127
+
128
+ void *rank_data;
129
+ size_t rank_data_sz = 16 * 1024 * 1024;
130
+ CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
131
+ std::vector<int64_t> offsets(nRanks, 0);
132
+ vllm::CustomAllreduce fa(buffer, rank_data, rank_data_sz, data_handles,
133
+ offsets, myRank);
134
+ auto *self_data =
135
+ reinterpret_cast<T *>(reinterpret_cast<char *>(buffer) +
136
+ sizeof(vllm::Metadata) + data_size * sizeof(T));
137
+ // hack buffer registration
138
+ {
139
+ std::vector<std::string> handles;
140
+ handles.reserve(nRanks);
141
+ for (int i = 0; i < nRanks; i++) {
142
+ char *begin = (char *)&data_handles[i];
143
+ char *end = (char *)&data_handles[i + 1];
144
+ handles.emplace_back(begin, end);
145
+ }
146
+ std::vector<int64_t> offsets(
147
+ nRanks, sizeof(vllm::Metadata) + data_size * sizeof(T));
148
+ fa.register_buffer(handles, offsets, self_data);
149
+ }
150
+
151
+ double *ground_truth;
152
+ CUDACHECK(cudaMallocHost(&ground_truth, data_size * sizeof(double)));
153
+ curandState_t *states;
154
+ CUDACHECK(cudaMalloc(&states, sizeof(curandState_t) * nRanks * data_size));
155
+ init_rand<<<108, 1024, 0, stream>>>(states, data_size, nRanks);
156
+ gen_data<T><<<108, 1024, 0, stream>>>(states, self_data, ground_truth, myRank,
157
+ nRanks, data_size);
158
+ CUDACHECK(cudaMemcpyAsync(self_data_copy, self_data, data_size * sizeof(T),
159
+ cudaMemcpyDeviceToDevice, stream));
160
+ cudaEvent_t start, stop;
161
+ CUDACHECK(cudaEventCreate(&start));
162
+ CUDACHECK(cudaEventCreate(&stop));
163
+
164
+ ncclDataType_t ncclDtype;
165
+ if (std::is_same<T, half>::value) {
166
+ ncclDtype = ncclFloat16;
167
+ } else if (std::is_same<T, nv_bfloat16>::value) {
168
+ ncclDtype = ncclBfloat16;
169
+ } else {
170
+ ncclDtype = ncclFloat;
171
+ }
172
+
173
+ dummy_kernel<<<1, 1, 0, stream>>>();
174
+ constexpr int warmup_iters = 5;
175
+ constexpr int num_iters = 25;
176
+ // warmup
177
+ for (int i = 0; i < warmup_iters; i++) {
178
+ NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
179
+ stream));
180
+ }
181
+ CUDACHECK(cudaEventRecord(start, stream));
182
+ for (int i = 0; i < num_iters; i++) {
183
+ NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
184
+ stream));
185
+ }
186
+ CUDACHECK(cudaEventRecord(stop, stream));
187
+ CUDACHECK(cudaStreamSynchronize(stream));
188
+ float allreduce_ms = 0;
189
+ cudaEventElapsedTime(&allreduce_ms, start, stop);
190
+
191
+ // if (myRank == 1) dummy_kernel<<<1, 1, 0, stream>>>();
192
+ // set_data<T><<<16, 1024, 0, stream>>>(self_data, data_size, myRank);
193
+
194
+ dummy_kernel<<<1, 1, 0, stream>>>();
195
+ // warm up
196
+ for (int i = 0; i < warmup_iters; i++) {
197
+ fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
198
+ }
199
+ CUDACHECK(cudaEventRecord(start, stream));
200
+ for (int i = 0; i < num_iters; i++) {
201
+ fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
202
+ }
203
+ CUDACHECK(cudaEventRecord(stop, stream));
204
+ CUDACHECK(cudaStreamSynchronize(stream));
205
+
206
+ float duration_ms = 0;
207
+ cudaEventElapsedTime(&duration_ms, start, stop);
208
+ if (myRank == 0)
209
+ printf(
210
+ "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
211
+ "time:%.2fus\n",
212
+ myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
213
+ duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
214
+
215
+ // And wait for all the queued up work to complete
216
+ CUDACHECK(cudaStreamSynchronize(stream));
217
+
218
+ NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
219
+ ncclSum, comm, stream));
220
+
221
+ double *nccl_result, *my_result;
222
+ CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
223
+ CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
224
+
225
+ convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
226
+ my_result, data_size);
227
+ CUDACHECK(cudaStreamSynchronize(stream));
228
+
229
+ for (unsigned long j = 0; j < data_size; j++) {
230
+ auto diff = abs(nccl_result[j] - my_result[j]);
231
+ if (diff >= 1e-2) {
232
+ printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
233
+ myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
234
+ break;
235
+ }
236
+ }
237
+
238
+ long double nccl_diffs = 0.0;
239
+ long double my_diffs = 0.0;
240
+ for (int j = 0; j < data_size; j++) {
241
+ nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
242
+ my_diffs += abs(my_result[j] - ground_truth[j]);
243
+ }
244
+ if (myRank == 0)
245
+ std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
246
+ << " me: " << my_diffs / data_size << std::endl;
247
+
248
+ CUDACHECK(cudaFree(result));
249
+ CUDACHECK(cudaFree(self_data_copy));
250
+ CUDACHECK(cudaFree(rank_data));
251
+ CUDACHECK(cudaFree(buffer));
252
+ CUDACHECK(cudaFree(states));
253
+ CUDACHECK(cudaFreeHost(ground_truth));
254
+ CUDACHECK(cudaFreeHost(nccl_result));
255
+ CUDACHECK(cudaFreeHost(my_result));
256
+ CUDACHECK(cudaStreamDestroy(stream));
257
+}
258
+
259
+int main(int argc, char **argv) {
260
+ int nRanks, myRank;
261
+ MPICHECK(MPI_Init(&argc, &argv));
262
+ MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
263
+ MPICHECK(MPI_Comm_size(MPI_COMM_WORLD, &nRanks));
264
+ CUDACHECK(cudaSetDevice(myRank));
265
+ ncclUniqueId id;
266
+ ncclComm_t comm;
267
+ if (myRank == 0) ncclGetUniqueId(&id);
268
+ MPICHECK(MPI_Bcast(static_cast<void *>(&id), sizeof(id), MPI_BYTE, 0,
269
+ MPI_COMM_WORLD));
270
+ NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
271
+
272
+ cudaProfilerStart();
273
+ // for (int threads : {256, 512}) {
274
+ // for (int block_limit = 16; block_limit < 112; block_limit += 4) {
275
+ // run<half>(myRank, nRanks, comm, threads, block_limit, 4096 * 1024);
276
+ // }
277
+ // }
278
+ for (int sz = 512; sz <= (32 << 20); sz *= 2) {
279
+ run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 50);
280
+ }
281
+
282
+ cudaProfilerStop();
283
+ return EXIT_SUCCESS;
284
+}
csrc/ops.h
+22
@@ -97,3 +97,25 @@ torch::Tensor gptq_gemm(
97
void gptq_shuffle(
98
torch::Tensor q_weight,
99
torch::Tensor q_perm);
100
+
101
+
102
+#ifndef USE_ROCM
103
+using fptr_t = uint64_t;
104
+fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
105
+ const std::vector<std::string> &handles,
106
+ const std::vector<int64_t> &offsets, int rank,
107
+ bool full_nvlink);
108
+bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
109
+ bool full_nvlink);
110
+void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out);
111
+void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor ®_buffer,
112
+ torch::Tensor &out);
113
+void dispose(fptr_t _fa);
114
+int meta_size();
115
+void register_buffer(fptr_t _fa, torch::Tensor &t,
116
+ const std::vector<std::string> &handles,
117
+ const std::vector<int64_t> &offsets);
118
+std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(fptr_t _fa);
119
+void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
120
+ const std::vector<std::vector<int64_t>> &offsets);
121
+#endif
csrc/pybind.cpp
+16
@@ -88,4 +88,20 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
88
&get_max_shared_memory_per_block_device_attribute,
89
"Gets the maximum shared memory per block device attribute.");
90
91
+#ifndef USE_ROCM
92
+ // Custom all-reduce kernels
93
+ pybind11::module custom_ar = m.def_submodule("custom_ar", "custom allreduce");
94
+ custom_ar.def("init_custom_ar", &init_custom_ar, "init_custom_ar");
95
+ custom_ar.def("should_custom_ar", &should_custom_ar, "should_custom_ar");
96
+ custom_ar.def("all_reduce_reg", &all_reduce_reg, "all_reduce_reg");
97
+ custom_ar.def("all_reduce_unreg", &all_reduce_unreg, "all_reduce_unreg");
98
+ custom_ar.def("dispose", &dispose, "dispose");
99
+ custom_ar.def("meta_size", &meta_size, "meta_size");
100
+ custom_ar.def("register_buffer", ®ister_buffer, "register_buffer");
101
+ custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta,
102
+ "get_graph_buffer_ipc_meta");
103
+ custom_ar.def("register_graph_buffers", ®ister_graph_buffers,
104
+ "register_graph_buffers");
105
+#endif
106
+
107
}
requirements.txt
+1
@@ -10,3 +10,4 @@ fastapi
10
uvicorn[standard]
11
pydantic >= 2.0 # Required for OpenAI server.
12
aioprometheus[starlette]
13
+pynvml == 11.5.0
setup.py
+4
-2
@@ -51,8 +51,8 @@ if _is_hip():
51
"Cannot find ROCM_HOME. ROCm must be available to build the package."
52
)
53
NVCC_FLAGS += ["-DUSE_ROCM"]
54
- NVCC_FLAGS += [f"-U__HIP_NO_HALF_CONVERSIONS__"]
55
- NVCC_FLAGS += [f"-U__HIP_NO_HALF_OPERATORS__"]
54
+ NVCC_FLAGS += ["-U__HIP_NO_HALF_CONVERSIONS__"]
55
+ NVCC_FLAGS += ["-U__HIP_NO_HALF_OPERATORS__"]
56
57
if _is_cuda() and CUDA_HOME is None:
58
raise RuntimeError(
@@ -307,6 +307,7 @@ vllm_extension_sources = [
307
308
if _is_cuda():
309
vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
310
+ vllm_extension_sources.append("csrc/custom_all_reduce.cu")
311
312
if not _is_neuron():
313
vllm_extension = CUDAExtension(
@@ -316,6 +317,7 @@ if not _is_neuron():
317
"cxx": CXX_FLAGS,
318
"nvcc": NVCC_FLAGS,
319
},
320
+ libraries=["cuda"] if _is_cuda() else [],
321
)
322
ext_modules.append(vllm_extension)
323
tests/distributed/test_comm_ops.py
+3
-27
@@ -6,25 +6,13 @@ import pytest
6
import torch
7
import ray
8
9
-from vllm.config import ParallelConfig
10
-from vllm.utils import get_open_port
9
from vllm.model_executor.parallel_utils.communication_op import (
10
tensor_model_parallel_all_reduce,
11
tensor_model_parallel_all_gather,
12
broadcast_tensor_dict,
13
)
16
-from vllm.worker.worker import _init_distributed_environment
17
-
18
-
19
-def init_test_distributed_environment(pipeline_parallel_size: int,
20
- tensor_parallel_size: int, rank: int,
21
- distributed_init_port: str):
22
- parallel_config = ParallelConfig(pipeline_parallel_size,
23
- tensor_parallel_size,
24
- worker_use_ray=True)
25
- distributed_init_method = f"tcp://localhost:{distributed_init_port}"
26
- _init_distributed_environment(parallel_config, rank,
27
- distributed_init_method)
14
+from vllm.test_utils import (init_test_distributed_environment,
15
+ multi_process_tensor_parallel)
16
17
18
@ray.remote(num_gpus=1, max_calls=1)
@@ -101,16 +89,4 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
89
broadcast_tensor_dict_test_worker
90
])
91
def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
104
- # Using ray helps debugging the error when it failed
105
- # as compared to multiprocessing.
106
- ray.init()
107
-
108
- distributed_init_port = get_open_port()
109
- refs = []
110
- for rank in range(tensor_parallel_size):
111
- refs.append(
112
- test_target.remote(tensor_parallel_size, rank,
113
- distributed_init_port))
114
- ray.get(refs)
115
-
116
- ray.shutdown()
92
+ multi_process_tensor_parallel(tensor_parallel_size, test_target)
tests/distributed/test_custom_all_reduce.py
new
+85
@@ -0,0 +1,85 @@
1
+import random
2
+
3
+import os
4
+import pytest
5
+import ray
6
+import torch
7
+import torch.distributed as dist
8
+
9
+from vllm.model_executor.parallel_utils import custom_all_reduce as custom_ar
10
+from vllm.model_executor.parallel_utils.communication_op import (
11
+ tensor_model_parallel_all_reduce)
12
+from vllm.test_utils import (init_test_distributed_environment,
13
+ multi_process_tensor_parallel)
14
+
15
+random.seed(42)
16
+test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
17
+for i, v in enumerate(test_sizes):
18
+ test_sizes[i] -= v % 8
19
+
20
+
21
+@ray.remote(num_gpus=1, max_calls=1)
22
+def graph_allreduce(world_size, rank, distributed_init_port):
23
+ del os.environ["CUDA_VISIBLE_DEVICES"]
24
+ device = torch.device(f"cuda:{rank}")
25
+ torch.cuda.set_device(device)
26
+ init_test_distributed_environment(1, world_size, rank,
27
+ distributed_init_port)
28
+
29
+ custom_ar.init_custom_ar()
30
+ for sz in test_sizes:
31
+ for dtype in [torch.float32, torch.float16, torch.bfloat16]:
32
+ with custom_ar.capture():
33
+ # use integers so result matches NCCL exactly
34
+ inp1 = torch.randint(1,
35
+ 16, (sz, ),
36
+ dtype=dtype,
37
+ device=torch.cuda.current_device())
38
+ inp2 = torch.randint(1,
39
+ 16, (sz, ),
40
+ dtype=dtype,
41
+ device=torch.cuda.current_device())
42
+ torch.cuda.synchronize()
43
+ graph = torch.cuda.CUDAGraph()
44
+ with torch.cuda.graph(graph):
45
+ out1 = tensor_model_parallel_all_reduce(inp1)
46
+ # the input buffer is immediately modified to test
47
+ # synchronization
48
+ dist.all_reduce(inp1)
49
+ out2 = tensor_model_parallel_all_reduce(inp2)
50
+ dist.all_reduce(inp2)
51
+ graph.replay()
52
+ assert torch.allclose(out1, inp1)
53
+ assert torch.allclose(out2, inp2)
54
+
55
+
56
+@ray.remote(num_gpus=1, max_calls=1)
57
+def eager_allreduce(world_size, rank, distributed_init_port):
58
+ del os.environ["CUDA_VISIBLE_DEVICES"]
59
+ device = torch.device(f"cuda:{rank}")
60
+ torch.cuda.set_device(device)
61
+ init_test_distributed_environment(1, world_size, rank,
62
+ distributed_init_port)
63
+
64
+ sz = 1024
65
+ custom_ar.init_custom_ar()
66
+ fa = custom_ar.get_handle()
67
+ inp = torch.ones(sz, dtype=torch.float32, device=device)
68
+ out = fa.all_reduce_unreg(inp)
69
+ assert torch.allclose(out, inp * world_size)
70
+
71
+ inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
72
+ out = fa.all_reduce_unreg(inp)
73
+ assert torch.allclose(out, inp * world_size)
74
+
75
+
76
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
77
+ reason="Need at least 2 GPUs to run the test.")
78
+@pytest.mark.parametrize("tensor_parallel_size", [2])
79
+@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
80
+def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
81
+ multi_process_tensor_parallel(tensor_parallel_size, test_target)
82
+
83
+
84
+if __name__ == "__main__":
85
+ multi_process_tensor_parallel(2, graph_allreduce)
vllm/config.py
+14
@@ -328,6 +328,8 @@ class ParallelConfig:
328
worker_use_ray: Whether to use Ray for model workers. Will be set to
329
True if either pipeline_parallel_size or tensor_parallel_size is
330
greater than 1.
331
+ disable_custom_all_reduce: Disable the custom all-reduce kernel and
332
+ fall back to NCCL.
333
"""
334
335
def __init__(
@@ -336,11 +338,13 @@ class ParallelConfig:
338
tensor_parallel_size: int,
339
worker_use_ray: bool,
340
max_parallel_loading_workers: Optional[int] = None,
341
+ disable_custom_all_reduce: bool = False,
342
) -> None:
343
self.pipeline_parallel_size = pipeline_parallel_size
344
self.tensor_parallel_size = tensor_parallel_size
345
self.worker_use_ray = worker_use_ray
346
self.max_parallel_loading_workers = max_parallel_loading_workers
347
+ self.disable_custom_all_reduce = disable_custom_all_reduce
348
349
self.world_size = pipeline_parallel_size * tensor_parallel_size
350
if self.world_size > 1:
@@ -351,6 +355,16 @@ class ParallelConfig:
355
if self.pipeline_parallel_size > 1:
356
raise NotImplementedError(
357
"Pipeline parallelism is not supported yet.")
358
+ if is_hip():
359
+ self.disable_custom_all_reduce = True
360
+ logger.info(
361
+ "Disabled the custom all-reduce kernel because it is not "
362
+ "supported on AMD GPUs.")
363
+ elif self.pipeline_parallel_size > 1:
364
+ self.disable_custom_all_reduce = True
365
+ logger.info(
366
+ "Disabled the custom all-reduce kernel because it is not "
367
+ "supported with pipeline parallelism.")
368
369
370
class SchedulerConfig:
vllm/engine/arg_utils.py
+7
-1
@@ -35,6 +35,7 @@ class EngineArgs:
35
quantization: Optional[str] = None
36
enforce_eager: bool = False
37
max_context_len_to_capture: int = 8192
38
+ disable_custom_all_reduce: bool = False
39
enable_lora: bool = False
40
max_loras: int = 1
41
max_lora_rank: int = 16
@@ -208,6 +209,10 @@ class EngineArgs:
209
help='maximum context length covered by CUDA '
210
'graphs. When a sequence has context length '
211
'larger than this, we fall back to eager mode.')
212
+ parser.add_argument('--disable-custom-all-reduce',
213
+ action='store_true',
214
+ default=EngineArgs.disable_custom_all_reduce,
215
+ help='See ParallelConfig')
216
# LoRA related configs
217
parser.add_argument('--enable-lora',
218
action='store_true',
@@ -269,7 +274,8 @@ class EngineArgs:
274
parallel_config = ParallelConfig(self.pipeline_parallel_size,
275
self.tensor_parallel_size,
276
self.worker_use_ray,
272
- self.max_parallel_loading_workers)
277
+ self.max_parallel_loading_workers,
278
+ self.disable_custom_all_reduce)
279
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
280
self.max_num_seqs,
281
model_config.max_model_len,
vllm/engine/llm_engine.py
+1
@@ -82,6 +82,7 @@ class LLMEngine:
82
f"download_dir={model_config.download_dir!r}, "
83
f"load_format={model_config.load_format}, "
84
f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
85
+ f"disable_custom_all_reduce={parallel_config.disable_custom_all_reduce}, "
86
f"quantization={model_config.quantization}, "
87
f"enforce_eager={model_config.enforce_eager}, "
88
f"seed={model_config.seed})")
vllm/entrypoints/llm.py
+3
@@ -64,6 +64,7 @@ class LLM:
64
max_context_len_to_capture: Maximum context len covered by CUDA graphs.
65
When a sequence has context length larger than this, we fall back
66
to eager mode.
67
+ disable_custom_all_reduce: See ParallelConfig
68
"""
69
70
def __init__(
@@ -82,6 +83,7 @@ class LLM:
83
swap_space: int = 4,
84
enforce_eager: bool = False,
85
max_context_len_to_capture: int = 8192,
86
+ disable_custom_all_reduce: bool = False,
87
**kwargs,
88
) -> None:
89
if "disable_log_stats" not in kwargs:
@@ -101,6 +103,7 @@ class LLM:
103
swap_space=swap_space,
104
enforce_eager=enforce_eager,
105
max_context_len_to_capture=max_context_len_to_capture,
106
+ disable_custom_all_reduce=disable_custom_all_reduce,
107
**kwargs,
108
)
109
self.llm_engine = LLMEngine.from_engine_args(engine_args)
vllm/model_executor/parallel_utils/communication_op.py
+12
-2
@@ -10,17 +10,27 @@ from vllm.model_executor.parallel_utils.parallel_state import (
10
get_tensor_model_parallel_world_size,
11
get_tensor_model_parallel_group,
12
)
13
+from vllm.model_executor.parallel_utils.custom_all_reduce import custom_all_reduce
14
15
16
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
17
"""All-reduce the input tensor across model parallel group.
18
18
- NOTE: This operation is applied in-place on the input tensor.
19
+ NOTE: This operation will be applied in-place on the input tensor if
20
+ disable_custom_all_reduce is set to True. Otherwise, this operation may or
21
+ may not be applied in place depending on whether custom all reduce is
22
+ invoked for a particular tensor, which further depends on the tensor size
23
+ and GPU topology.
24
+
25
+ TLDR: always assume this function modifies its input, but use the return
26
+ value as the output.
27
"""
28
# Bypass the function if we are using only 1 GPU.
29
if get_tensor_model_parallel_world_size() == 1:
30
return input_
23
- # All-reduce.
31
+ out = custom_all_reduce(input_)
32
+ if out is not None:
33
+ return out
34
torch.distributed.all_reduce(input_,
35
group=get_tensor_model_parallel_group())
36
return input_
vllm/model_executor/parallel_utils/custom_all_reduce.py
new
+223
@@ -0,0 +1,223 @@
1
+from contextlib import contextmanager
2
+from typing import Optional
3
+
4
+import torch
5
+import torch.distributed as dist
6
+
7
+from vllm.logger import init_logger
8
+from vllm.model_executor.parallel_utils.parallel_state import (
9
+ get_tensor_model_parallel_world_size, get_tensor_model_parallel_rank)
10
+
11
+try:
12
+ from vllm._C import custom_ar
13
+ import pynvml
14
+except ImportError:
15
+ # For AMD GPUs
16
+ custom_ar = None
17
+ pynvml = None
18
+
19
+logger = init_logger(__name__)
20
+
21
+_CA_HANDLE = None
22
+_IS_CAPTURING = False
23
+_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
24
+
25
+
26
+def init_custom_ar() -> None:
27
+ global _CA_HANDLE
28
+ if _CA_HANDLE is not None:
29
+ return
30
+ rank = get_tensor_model_parallel_rank()
31
+ world_size = get_tensor_model_parallel_world_size()
32
+ if world_size not in _SUPPORTED_WORLD_SIZES:
33
+ logger.warn(
34
+ "Custom allreduce is disabled due to an unsupported world size: "
35
+ "%d. Supported world sizes: %s. To slience this warning, specify"
36
+ "disable_custom_all_reduce=True explicitly.", world_size,
37
+ str(_SUPPORTED_WORLD_SIZES))
38
+ return
39
+ if not _can_p2p(rank, world_size):
40
+ logger.warn(
41
+ "Custom allreduce is disabled because your platform lacks GPU P2P"
42
+ " capability. To slience this warning, specify"
43
+ "disable_custom_all_reduce=True explicitly.")
44
+ return
45
+ _CA_HANDLE = CustomAllreduce(rank, world_size)
46
+
47
+
48
+def begin_capture() -> None:
49
+ global _IS_CAPTURING
50
+ _IS_CAPTURING = True
51
+
52
+
53
+def end_capture() -> None:
54
+ global _IS_CAPTURING
55
+ _IS_CAPTURING = False
56
+
57
+
58
+def is_capturing() -> bool:
59
+ return _IS_CAPTURING and _CA_HANDLE is not None
60
+
61
+
62
+def get_handle() -> Optional["CustomAllreduce"]:
63
+ return _CA_HANDLE
64
+
65
+
66
+@contextmanager
67
+def capture():
68
+ try:
69
+ begin_capture()
70
+ yield
71
+ finally:
72
+ end_capture()
73
+ handle = get_handle()
74
+ if handle is not None:
75
+ handle.register_graph_buffers()
76
+
77
+
78
+def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
79
+ ca_handle = get_handle()
80
+ # when custom allreduce is disabled, this will be None
81
+ if ca_handle is None:
82
+ return
83
+ if is_capturing():
84
+ if torch.cuda.is_current_stream_capturing():
85
+ if ca_handle.should_custom_ar(input):
86
+ return ca_handle.all_reduce_reg(input)
87
+ else:
88
+ if ca_handle.should_custom_ar(input):
89
+ # if warm up, mimic the allocation pattern
90
+ # since custom allreduce is out-of-place
91
+ return torch.empty_like(input)
92
+ else:
93
+ # note: outside of cuda graph context,
94
+ # custom allreduce incurs a cost of cudaMemcpy, which should
95
+ # be small(<=1% of overall latency) compared to the performance
96
+ # gains of using custom kernels
97
+ if ca_handle.should_custom_ar(input):
98
+ return ca_handle.all_reduce_unreg(input)
99
+
100
+
101
+@contextmanager
102
+def _nvml():
103
+ try:
104
+ pynvml.nvmlInit()
105
+ yield
106
+ finally:
107
+ pynvml.nvmlShutdown()
108
+
109
+
110
+# query if the set of gpus are fully connected by nvlink (1 hop)
111
+@_nvml()
112
+def _is_full_nvlink(rank, world_size):
113
+ handle = pynvml.nvmlDeviceGetHandleByIndex(rank)
114
+ for i in range(world_size):
115
+ if i != rank:
116
+ try:
117
+ link_state = pynvml.nvmlDeviceGetNvLinkState(handle, i)
118
+ if not link_state:
119
+ return False
120
+ except pynvml.NVMLError as error:
121
+ logger.info(
122
+ f"NVLink detection failed with message \"{str(error)}\". "
123
+ "This is normal if your machine has no NVLink equipped")
124
+ return False
125
+ return True
126
+
127
+
128
+def _can_p2p(rank: int, world_size: int) -> bool:
129
+ for i in range(world_size):
130
+ if i == rank:
131
+ continue
132
+ if not torch.cuda.can_device_access_peer(rank, i):
133
+ return False
134
+ return True
135
+
136
+
137
+class CustomAllreduce:
138
+
139
+ # max_size: max supported allreduce size
140
+ def __init__(self, rank, world_size, max_size=8192 * 1024) -> None:
141
+ # buffers memory are owned by this Python class and passed to C++
142
+ # meta data composes of two parts: meta data for synchronization
143
+ # (256 bytes) and a temporary buffer for storing intermediate
144
+ # allreduce results.
145
+ self.meta = torch.zeros(custom_ar.meta_size() + max_size,
146
+ dtype=torch.uint8,
147
+ device="cuda")
148
+ # This is a pre-registered IPC buffer. In eager mode, input tensors
149
+ # are first copied into this buffer before allreduce is performed
150
+ self.buffer = torch.empty(max_size, dtype=torch.uint8, device="cuda")
151
+ # This is a buffer for storing the tuples of pointers pointing to
152
+ # IPC buffers from all ranks. Each registered tuple has size of
153
+ # 8*world_size bytes where world_size is at most 8. Allocating 8MB
154
+ # is enough for 131072 such tuples. The largest model I've seen only
155
+ # needs less than 10000 of registered tuples.
156
+ self.rank_data = torch.empty(8 * 1024 * 1024,
157
+ dtype=torch.uint8,
158
+ device="cuda")
159
+ self.max_size = max_size
160
+ self.world_size = world_size
161
+ handles, offsets = self._get_ipc_meta(self.meta)
162
+ self.full_nvlink = _is_full_nvlink(rank, world_size)
163
+ self._ptr = custom_ar.init_custom_ar(self.meta, self.rank_data,
164
+ handles, offsets, rank,
165
+ self.full_nvlink)
166
+ self.fast_cond = self.full_nvlink or world_size <= 2
167
+ self.register_buffer(self.buffer)
168
+
169
+ def _get_ipc_meta(self, inp: torch.Tensor):
170
+ data = inp.untyped_storage()._share_cuda_()
171
+ shard_data = (
172
+ data[1], # ipc handle to base ptr
173
+ data[3], # offset of base ptr
174
+ )
175
+ return self._gather_ipc_meta(shard_data)
176
+
177
+ def _gather_ipc_meta(self, shard_data):
178
+ all_data = [None] * self.world_size
179
+ dist.all_gather_object(all_data, shard_data)
180
+
181
+ handles = []
182
+ offsets = []
183
+ for i in range(len(all_data)):
184
+ handles.append(all_data[i][0])
185
+ offsets.append(all_data[i][1])
186
+ return handles, offsets
187
+
188
+ def register_buffer(self, inp: torch.Tensor):
189
+ handles, offsets = self._get_ipc_meta(inp)
190
+ custom_ar.register_buffer(self._ptr, inp, handles, offsets)
191
+
192
+ def register_graph_buffers(self):
193
+ handle, offset = custom_ar.get_graph_buffer_ipc_meta(self._ptr)
194
+ handles, offsets = self._gather_ipc_meta((bytes(handle), offset))
195
+ logger.info("Registering %d cuda graph addresses", len(offset))
196
+ custom_ar.register_graph_buffers(self._ptr, handles, offsets)
197
+
198
+ def should_custom_ar(self, inp: torch.Tensor):
199
+ return custom_ar.should_custom_ar(inp, self.max_size, self.world_size,
200
+ self.full_nvlink)
201
+
202
+ # all reduce, assuming inp tensor is IPC registered with register_buffer,
203
+ # or, in the context of cuda graphs, register_graph_buffers
204
+ def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None):
205
+ if out is None:
206
+ out = torch.empty_like(inp)
207
+ custom_ar.all_reduce_reg(self._ptr, inp, out)
208
+ return out
209
+
210
+ # all reduce, assuming inp tensor is NOT IPC registered
211
+ def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None):
212
+ if out is None:
213
+ out = torch.empty_like(inp)
214
+ custom_ar.all_reduce_unreg(self._ptr, inp, self.buffer, out)
215
+ return out
216
+
217
+ def close(self):
218
+ if self._ptr:
219
+ custom_ar.dispose(self._ptr)
220
+ self._ptr = 0
221
+
222
+ def __del__(self):
223
+ self.close()
vllm/test_utils.py
new
+38
@@ -0,0 +1,38 @@
1
+import ray
2
+
3
+from vllm.config import ParallelConfig
4
+from vllm.utils import get_open_port
5
+from vllm.worker.worker import init_distributed_environment
6
+
7
+
8
+def init_test_distributed_environment(
9
+ pipeline_parallel_size: int,
10
+ tensor_parallel_size: int,
11
+ rank: int,
12
+ distributed_init_port: str,
13
+) -> None:
14
+ parallel_config = ParallelConfig(pipeline_parallel_size,
15
+ tensor_parallel_size,
16
+ worker_use_ray=True)
17
+ distributed_init_method = f"tcp://localhost:{distributed_init_port}"
18
+ init_distributed_environment(parallel_config, rank,
19
+ distributed_init_method)
20
+
21
+
22
+def multi_process_tensor_parallel(
23
+ tensor_parallel_size: int,
24
+ test_target,
25
+) -> None:
26
+ # Using ray helps debugging the error when it failed
27
+ # as compared to multiprocessing.
28
+ ray.init()
29
+
30
+ distributed_init_port = get_open_port()
31
+ refs = []
32
+ for rank in range(tensor_parallel_size):
33
+ refs.append(
34
+ test_target.remote(tensor_parallel_size, rank,
35
+ distributed_init_port))
36
+ ray.get(refs)
37
+
38
+ ray.shutdown()
vllm/worker/model_runner.py
+31
-29
@@ -10,6 +10,7 @@ from vllm.logger import init_logger
10
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11
from vllm.model_executor.parallel_utils.communication_op import (
12
broadcast_tensor_dict)
13
+from vllm.model_executor.parallel_utils import custom_all_reduce
14
from vllm.sampling_params import SamplingParams, SamplingType
15
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
16
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
@@ -651,37 +652,38 @@ class ModelRunner:
652
653
# NOTE: Capturing the largest batch size first may help reduce the
654
# memory usage of CUDA graph.
654
- for batch_size in reversed(batch_size_capture_list):
655
- # Create dummy input_metadata.
656
- input_metadata = InputMetadata(
657
- is_prompt=False,
658
- slot_mapping=slot_mapping[:batch_size],
659
- prompt_lens=None,
660
- max_seq_len=None,
661
- start_loc=None,
662
- max_context_len=self.max_context_len_to_capture,
663
- context_lens=context_lens[:batch_size],
664
- block_tables=block_tables[:batch_size],
665
- use_cuda_graph=True,
666
- )
667
-
668
- if self.lora_config:
669
- lora_mapping = LoRAMapping(
670
- [0] * batch_size,
671
- [0] * batch_size,
655
+ with custom_all_reduce.capture():
656
+ for batch_size in reversed(batch_size_capture_list):
657
+ # Create dummy input_metadata.
658
+ input_metadata = InputMetadata(
659
+ is_prompt=False,
660
+ slot_mapping=slot_mapping[:batch_size],
661
+ prompt_lens=None,
662
+ max_seq_len=None,
663
+ start_loc=None,
664
+ max_context_len=self.max_context_len_to_capture,
665
+ context_lens=context_lens[:batch_size],
666
+ block_tables=block_tables[:batch_size],
667
+ use_cuda_graph=True,
668
)
673
- self.set_active_loras(set(), lora_mapping)
669
675
- graph_runner = CUDAGraphRunner(self.model)
676
- graph_runner.capture(
677
- input_tokens[:batch_size],
678
- input_positions[:batch_size],
679
- kv_caches,
680
- input_metadata,
681
- memory_pool=self.graph_memory_pool,
682
- )
683
- self.graph_memory_pool = graph_runner.graph.pool()
684
- self.graph_runners[batch_size] = graph_runner
670
+ if self.lora_config:
671
+ lora_mapping = LoRAMapping(
672
+ [0] * batch_size,
673
+ [0] * batch_size,
674
+ )
675
+ self.set_active_loras(set(), lora_mapping)
676
+
677
+ graph_runner = CUDAGraphRunner(self.model)
678
+ graph_runner.capture(
679
+ input_tokens[:batch_size],
680
+ input_positions[:batch_size],
681
+ kv_caches,
682
+ input_metadata,
683
+ memory_pool=self.graph_memory_pool,
684
+ )
685
+ self.graph_memory_pool = graph_runner.graph.pool()
686
+ self.graph_runners[batch_size] = graph_runner
687
688
end_time = time.perf_counter()
689
elapsed_time = end_time - start_time
vllm/worker/worker.py
+6
-4
@@ -11,6 +11,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
11
from vllm.model_executor import set_random_seed
12
from vllm.model_executor.parallel_utils.communication_op import (
13
broadcast_tensor_dict)
14
+from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
15
from vllm.model_executor.parallel_utils.parallel_state import (
16
ensure_model_parallel_initialized)
17
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -78,9 +79,10 @@ class Worker:
79
_check_if_gpu_supports_dtype(self.model_config.dtype)
80
81
# Initialize the distributed environment.
81
- _init_distributed_environment(self.parallel_config, self.rank,
82
- self.distributed_init_method)
83
-
82
+ init_distributed_environment(self.parallel_config, self.rank,
83
+ self.distributed_init_method)
84
+ if not self.parallel_config.disable_custom_all_reduce:
85
+ init_custom_ar()
86
# Initialize the model.
87
set_random_seed(self.model_config.seed)
88
@@ -219,7 +221,7 @@ class Worker:
221
return self.model_runner.list_loras()
222
223
222
-def _init_distributed_environment(
224
+def init_distributed_environment(
225
parallel_config: ParallelConfig,
226
rank: int,
227
distributed_init_method: Optional[str] = None,