A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[BugFix] Some fixes for custom allreduce kernels (#2760)
Hanzhi Zhou committed
Mar 21, 2024 at 23:02 UTC
f721096d48a7e3b98dffcb9b400bf58989cef64d
6 files changed
+232
-250
csrc/custom_all_reduce.cu
+5
-5
@@ -29,7 +29,7 @@ fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
29
std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
30
}
31
return (fptr_t) new vllm::CustomAllreduce(
32
- reinterpret_cast<vllm::Metadata *>(meta.data_ptr()), rank_data.data_ptr(),
32
+ reinterpret_cast<vllm::Signal *>(meta.data_ptr()), rank_data.data_ptr(),
33
rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
34
}
35
@@ -62,9 +62,9 @@ bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
62
if (inp_size % 16 != 0) return false;
63
if (!_is_weak_contiguous(inp)) return false;
64
if (world_size == 2 || full_nvlink) return inp_size <= max_size;
65
- // 4 PCIE GPUs use 2 stage allreduce, and is only faster than NCCL when size
66
- // <= 512k
67
- return world_size <= 4 && inp_size <= 512 * 1024;
65
+ // for 4 or more non NVLink-capable GPUs, custom allreduce provides little
66
+ // performance improvement over NCCL.
67
+ return false;
68
}
69
70
void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
@@ -126,7 +126,7 @@ void dispose(fptr_t _fa) {
126
delete fa;
127
}
128
129
-int meta_size() { return sizeof(vllm::Metadata); }
129
+int meta_size() { return sizeof(vllm::Signal); }
130
131
void register_buffer(fptr_t _fa, torch::Tensor &t,
132
const std::vector<std::string> &handles,
csrc/custom_all_reduce.cuh
+75
-152
@@ -23,29 +23,17 @@
23
24
namespace vllm {
25
26
+constexpr int kMaxBlocks = 64;
27
+// note: we don't want to use atomics for signals because peer atomics are no
28
+// supported on PCIe links
29
struct Signal {
27
- alignas(64) union {
28
- uint64_t flag;
29
- unsigned char data[8];
30
- } start;
31
- alignas(64) union {
32
- uint64_t flag;
33
- unsigned char data[8];
34
- } end;
30
+ alignas(128) uint32_t start[kMaxBlocks][8];
31
+ alignas(128) uint32_t end[kMaxBlocks][8];
32
};
33
37
-struct Metadata {
38
- alignas(128) Signal sg;
39
- alignas(128) int counter;
40
-};
41
-static_assert(offsetof(Metadata, counter) == 128);
42
-static_assert(sizeof(Metadata) == 256);
43
-
34
struct __align__(16) RankData { const void *__restrict__ ptrs[8]; };
35
46
-struct RankSignals {
47
- volatile Signal *signals[8];
48
-};
36
+struct __align__(16) RankSignals { volatile Signal *signals[8]; };
37
38
// like std::array, but aligned
39
template <typename T, int sz>
@@ -135,70 +123,49 @@ DINLINE O downcast(array_t<float, O::size> val) {
123
}
124
}
125
138
-// compute flag at compile time
139
-__host__ __device__ constexpr uint64_t compute_flag(int ngpus) {
140
- auto m = std::numeric_limits<uint64_t>::max();
141
- return m >> ((8 - ngpus) * 8);
142
-}
143
-
126
+// This function is meant to be used as the first synchronization in the all
127
+// reduce kernel. Thus, it doesn't need to make any visibility guarantees for
128
+// prior memory accesses. Note: volatile writes will not be reordered against
129
+// other volatile writes.
130
template <int ngpus>
145
-DINLINE void start_sync(const RankSignals &sg, volatile Metadata *meta,
131
+DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
132
int rank) {
147
- constexpr auto FLAG = compute_flag(ngpus);
148
- if (blockIdx.x == 0) {
149
- if (threadIdx.x < ngpus)
150
- // simultaneously write to the corresponding byte to all other ranks.
151
- // Latency = 1 p2p write
152
- sg.signals[threadIdx.x]->start.data[rank] = 255;
153
- else if (threadIdx.x == 32)
154
- // reset
155
- meta->sg.end.flag = 0;
156
- }
157
- if (threadIdx.x == 0) {
158
- while (meta->sg.start.flag != FLAG)
133
+ if (threadIdx.x < ngpus) {
134
+ // reset flag for next time
135
+ self_sg->end[blockIdx.x][threadIdx.x] = 0;
136
+ // simultaneously write to the corresponding flag of all ranks.
137
+ // Latency = 1 p2p write
138
+ sg.signals[threadIdx.x]->start[blockIdx.x][rank] = 1;
139
+ // wait until we got true from all ranks
140
+ while (!self_sg->start[blockIdx.x][threadIdx.x])
141
;
142
}
143
__syncthreads();
144
}
145
146
+// This function is meant to be used as the second or the final synchronization
147
+// barrier in the all reduce kernel. If it's the final synchronization barrier,
148
+// we don't need to make any visibility guarantees for prior memory accesses.
149
template <int ngpus, bool final_sync = false>
165
-DINLINE void end_sync(const RankSignals &sg, volatile Metadata *meta,
150
+DINLINE void end_sync(const RankSignals &sg, volatile Signal *self_sg,
151
int rank) {
167
- constexpr auto FLAG = compute_flag(ngpus);
152
__syncthreads();
169
- __shared__ int num;
170
- if (threadIdx.x == 0) num = atomicAdd((int *)&meta->counter, 1);
171
- __syncthreads();
172
-
173
- // Only the last completing block can perform the end synchronization
174
- // This can ensures when the final busy wait ends, all ranks must have
175
- // finished reading each other's buffer.
176
- if (num == gridDim.x - 1) {
177
- if (threadIdx.x == 32) {
178
- // reset in a different warp
179
- meta->counter = 0;
180
- meta->sg.start.flag = 0;
181
- } else if (threadIdx.x < ngpus) {
182
- // simultaneously write to the corresponding byte to all other ranks.
183
- // Latency = 1 p2p write
184
- sg.signals[threadIdx.x]->end.data[rank] = 255;
185
- }
186
- // if this is the final sync, only one block needs it
187
- // because kernel exit can serve as sync
188
- if constexpr (final_sync) {
189
- if (threadIdx.x == 0) {
190
- while (meta->sg.end.flag != FLAG)
191
- ;
192
- }
193
- }
194
- }
195
- if constexpr (!final_sync) {
196
- if (threadIdx.x == 0) {
197
- while (meta->sg.end.flag != FLAG)
198
- ;
199
- }
200
- __syncthreads();
153
+ // eliminate the case that prior writes are not visible after signals become
154
+ // visible. Note that I did not managed to make this happen through a lot of
155
+ // testing. Might be the case that hardware provides stronger guarantee than
156
+ // the memory model.
157
+ if constexpr (!final_sync) __threadfence_system();
158
+ if (threadIdx.x < ngpus) {
159
+ // reset flag for next time
160
+ self_sg->start[blockIdx.x][threadIdx.x] = 0;
161
+ // simultaneously write to the corresponding flag of all ranks.
162
+ // Latency = 1 p2p write
163
+ sg.signals[threadIdx.x]->end[blockIdx.x][rank] = 1;
164
+ // wait until we got true from all ranks
165
+ while (!self_sg->end[blockIdx.x][threadIdx.x])
166
+ ;
167
}
168
+ if constexpr (!final_sync) __syncthreads();
169
}
170
171
template <typename P, int ngpus, typename A>
@@ -214,32 +181,32 @@ DINLINE P packed_reduce(const P *ptrs[], int idx) {
181
template <typename T, int ngpus>
182
__global__ void __launch_bounds__(512, 1)
183
cross_device_reduce_1stage(RankData *_dp, RankSignals sg,
217
- volatile Metadata *meta, T *__restrict__ result,
184
+ volatile Signal *self_sg, T *__restrict__ result,
185
int rank, int size) {
186
using P = typename packed_t<T>::P;
187
using A = typename packed_t<T>::A;
188
// note: we don't reorder the address so the accumulation order is the same
189
// for all ranks, ensuring bitwise identical results
190
auto dp = *_dp;
224
- start_sync<ngpus>(sg, meta, rank);
191
+ start_sync<ngpus>(sg, self_sg, rank);
192
// do the actual reduction
193
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
194
idx += gridDim.x * blockDim.x) {
195
((P *)result)[idx] =
196
packed_reduce<P, ngpus, A>((const P **)&dp.ptrs[0], idx);
197
}
231
- end_sync<ngpus, true>(sg, meta, rank);
198
+ end_sync<ngpus, true>(sg, self_sg, rank);
199
}
200
201
template <typename P>
202
DINLINE P *get_tmp_buf(volatile Signal *sg) {
236
- return (P *)(((Metadata *)sg) + 1);
203
+ return (P *)(((Signal *)sg) + 1);
204
}
205
206
template <typename T, int ngpus>
207
__global__ void __launch_bounds__(512, 1)
208
cross_device_reduce_2stage(RankData *_dp, RankSignals sg,
242
- volatile Metadata *meta, T *__restrict__ result,
209
+ volatile Signal *self_sg, T *__restrict__ result,
210
int rank, int size) {
211
int tid = blockIdx.x * blockDim.x + threadIdx.x;
212
int stride = gridDim.x * blockDim.x;
@@ -248,6 +215,7 @@ __global__ void __launch_bounds__(512, 1)
215
int part = size / ngpus;
216
int start = rank * part;
217
int end = rank == ngpus - 1 ? size : start + part;
218
+ int largest_part = part + size % ngpus;
219
const P *ptrs[ngpus];
220
P *tmps[ngpus];
221
#pragma unroll
@@ -257,75 +225,28 @@ __global__ void __launch_bounds__(512, 1)
225
tmps[i] = get_tmp_buf<P>(sg.signals[target]);
226
}
227
auto tmp_out = tmps[0];
260
- start_sync<ngpus>(sg, meta, rank);
228
+ start_sync<ngpus>(sg, self_sg, rank);
229
// stage 1: reduce scatter
230
for (int idx = start + tid; idx < end; idx += stride) {
231
tmp_out[idx - start] = packed_reduce<P, ngpus, A>(ptrs, idx);
232
}
265
- // Maybe TODO: replace this with per-block release-acquire
266
- // can save about 1-2us (not a lot though)
267
- end_sync<ngpus>(sg, meta, rank);
268
-
269
- // stage 2: allgather
270
- for (int idx = tid; idx < part; idx += stride) {
233
+ end_sync<ngpus>(sg, self_sg, rank);
234
+
235
+ // stage 2: allgather. Note: it's important to match the tid between
236
+ // the two stages, because visibility across devices is only guaranteed
237
+ // between threads that have the same tid. If thread i computes the sum of
238
+ // start + i in the first stage, then thread i also gathers start + i from all
239
+ // ranks.
240
+ for (int idx = tid; idx < largest_part; idx += stride) {
241
#pragma unroll
242
for (int i = 0; i < ngpus; i++) {
273
- int dst_idx = ((rank + i) % ngpus) * part + idx;
274
- ((P *)result)[dst_idx] = tmps[i][idx];
275
- }
276
- }
277
- // process the last larger partition
278
- int remaining = size - part * ngpus;
279
- if (tid < remaining) {
280
- int dst_idx = tid + part * ngpus;
281
- ((P *)result)[dst_idx] = get_tmp_buf<P>(sg.signals[ngpus - 1])[part + tid];
282
- }
283
-
284
- // faster than this
285
- // for (int idx = tid; idx < size; idx += stride) {
286
- // int target_rank = idx / part;
287
- // if (target_rank == ngpus) target_rank -= 1;
288
- // ((P *)result)[idx] = tmps[target_rank][idx - target_rank * part];
289
- // }
290
-}
291
-
292
-template <typename T, int ngpus>
293
-__global__ void __launch_bounds__(512, 1)
294
- cross_device_reduce_half_butterfly(RankData *_dp, RankSignals sg,
295
- volatile Metadata *meta,
296
- T *__restrict__ result, int rank,
297
- int size) {
298
- int tid = blockIdx.x * blockDim.x + threadIdx.x;
299
- int stride = gridDim.x * blockDim.x;
300
- using P = typename packed_t<T>::P;
301
- using A = typename packed_t<T>::A;
302
- auto tmp_out = get_tmp_buf<P>(sg.signals[rank]);
303
- constexpr int hg = ngpus / 2;
304
- // Actually not quite half butterfly.
305
- // This is an all-to-all within each group containing half of the ranks
306
- // followed by cross-group add. Equivalent to half butterfly when there
307
- // are 4 GPUs, a common case for PCIe cards like T4 and A10.
308
- const P *ptrs[hg];
309
- {
310
- int start = rank - rank % hg;
311
-#pragma unroll
312
- for (int i = 0; i < hg; i++) {
313
- ptrs[i] = (const P *)_dp->ptrs[i + start];
243
+ int gather_from_rank = ((rank + i) % ngpus);
244
+ if (gather_from_rank == ngpus - 1 || idx < part) {
245
+ int dst_idx = gather_from_rank * part + idx;
246
+ ((P *)result)[dst_idx] = tmps[i][idx];
247
+ }
248
}
249
}
316
- start_sync<ngpus>(sg, meta, rank);
317
- for (int idx = tid; idx < size; idx += stride) {
318
- tmp_out[idx] = packed_reduce<P, hg, A>(ptrs, idx);
319
- }
320
- end_sync<ngpus>(sg, meta, rank);
321
-
322
- auto src = get_tmp_buf<P>(sg.signals[(ngpus - 1) - rank % ngpus]);
323
- // do the cross group reduction
324
- for (int idx = tid; idx < size; idx += stride) {
325
- auto tmp = tmp_out[idx];
326
- packed_assign_add(tmp, src[idx]);
327
- ((P *)result)[idx] = tmp;
328
- }
250
}
251
252
using IPC_KEY = std::array<uint8_t, sizeof(cudaIpcMemHandle_t)>;
@@ -341,7 +262,7 @@ class CustomAllreduce {
262
// below are device pointers
263
RankSignals sg_;
264
std::unordered_map<void *, RankData *> buffers_;
344
- Metadata *meta_;
265
+ Signal *self_sg_;
266
267
// stores the registered device pointers from all ranks
268
RankData *d_rank_data_base_, *d_rank_data_end_;
@@ -352,32 +273,32 @@ class CustomAllreduce {
273
/**
274
* meta is a pointer to device metadata and temporary buffer for allreduce.
275
*
355
- * There's a total of sizeof(Metadata) of prefix before the actual data,
276
+ * There's a total of sizeof(Signal) of prefix before the actual data,
277
* so meta + 1 points to actual temporary buffer.
278
*
279
* note: this class does not own any device memory. Any required buffers
280
* are passed in from the constructor
281
*/
361
- CustomAllreduce(Metadata *meta, void *rank_data, size_t rank_data_sz,
282
+ CustomAllreduce(Signal *meta, void *rank_data, size_t rank_data_sz,
283
const cudaIpcMemHandle_t *handles,
284
const std::vector<int64_t> &offsets, int rank,
285
bool full_nvlink = true)
286
: rank_(rank),
287
world_size_(offsets.size()),
288
full_nvlink_(full_nvlink),
368
- meta_(meta),
289
+ self_sg_(meta),
290
d_rank_data_base_(reinterpret_cast<RankData *>(rank_data)),
291
d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
292
for (int i = 0; i < world_size_; i++) {
372
- Metadata *rank_meta;
293
+ Signal *rank_sg;
294
if (i != rank_) {
295
char *handle = open_ipc_handle(&handles[i]);
296
handle += offsets[i];
376
- rank_meta = (Metadata *)handle;
297
+ rank_sg = (Signal *)handle;
298
} else {
378
- rank_meta = meta_;
299
+ rank_sg = self_sg_;
300
}
380
- sg_.signals[i] = &rank_meta->sg;
301
+ sg_.signals[i] = rank_sg;
302
}
303
}
304
@@ -492,6 +413,10 @@ class CustomAllreduce {
413
"custom allreduce currently requires input length to be multiple "
414
"of " +
415
std::to_string(d));
416
+ if (block_limit > kMaxBlocks)
417
+ throw std::runtime_error("max supported block limit is " +
418
+ std::to_string(kMaxBlocks) + ". Got " +
419
+ std::to_string(block_limit));
420
421
RankData *ptrs;
422
cudaStreamCaptureStatus status;
@@ -512,9 +437,9 @@ class CustomAllreduce {
437
size /= d;
438
auto bytes = size * sizeof(typename packed_t<T>::P);
439
int blocks = std::min(block_limit, (size + threads - 1) / threads);
515
-#define KL(ngpus, name) \
516
- name<T, ngpus> \
517
- <<<blocks, threads, 0, stream>>>(ptrs, sg_, meta_, output, rank_, size);
440
+#define KL(ngpus, name) \
441
+ name<T, ngpus><<<blocks, threads, 0, stream>>>(ptrs, sg_, self_sg_, output, \
442
+ rank_, size);
443
#define REDUCE_CASE(ngpus) \
444
case ngpus: { \
445
if (world_size_ == 2) { \
@@ -526,8 +451,6 @@ class CustomAllreduce {
451
} else { \
452
KL(ngpus, cross_device_reduce_2stage); \
453
} \
529
- } else { \
530
- KL(ngpus, cross_device_reduce_half_butterfly); \
454
} \
455
break; \
456
}
@@ -556,7 +479,7 @@ class CustomAllreduce {
479
/**
480
* To inspect PTX/SASS, copy paste this header file to compiler explorer and add
481
a template instantiation:
559
- * template void CustomAllreduce::allreduce<half>(cudaStream_t, half *, half *,
560
- int, int, int);
482
+ * template void vllm::CustomAllreduce::allreduce<half>(cudaStream_t, half *,
483
+ half *, int, int, int);
484
*/
485
} // namespace vllm
csrc/custom_all_reduce_test.cu
+108
-76
@@ -92,7 +92,7 @@ __global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
92
93
template <typename T>
94
void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
95
- int data_size) {
95
+ int data_size, bool performance_test) {
96
T *result;
97
cudaStream_t stream;
98
CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
@@ -101,7 +101,7 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
101
102
cudaIpcMemHandle_t self_data_handle;
103
cudaIpcMemHandle_t data_handles[8];
104
- vllm::Metadata *buffer;
104
+ vllm::Signal *buffer;
105
T *self_data_copy;
106
/**
107
* Allocate IPC buffer
@@ -115,9 +115,9 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
115
* convenience.
116
*/
117
CUDACHECK(
118
- cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
119
- CUDACHECK(cudaMemset(buffer, 0,
120
- 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
118
+ cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
119
+ CUDACHECK(
120
+ cudaMemset(buffer, 0, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
121
CUDACHECK(cudaMalloc(&self_data_copy, data_size * sizeof(T)));
122
CUDACHECK(cudaIpcGetMemHandle(&self_data_handle, buffer));
123
@@ -133,7 +133,7 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
133
offsets, myRank);
134
auto *self_data =
135
reinterpret_cast<T *>(reinterpret_cast<char *>(buffer) +
136
- sizeof(vllm::Metadata) + data_size * sizeof(T));
136
+ sizeof(vllm::Signal) + data_size * sizeof(T));
137
// hack buffer registration
138
{
139
std::vector<std::string> handles;
@@ -143,8 +143,8 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
143
char *end = (char *)&data_handles[i + 1];
144
handles.emplace_back(begin, end);
145
}
146
- std::vector<int64_t> offsets(
147
- nRanks, sizeof(vllm::Metadata) + data_size * sizeof(T));
146
+ std::vector<int64_t> offsets(nRanks,
147
+ sizeof(vllm::Signal) + data_size * sizeof(T));
148
fa.register_buffer(handles, offsets, self_data);
149
}
150
@@ -169,81 +169,112 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
169
} else {
170
ncclDtype = ncclFloat;
171
}
172
+ double *nccl_result, *my_result;
173
+ CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
174
+ CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
175
+ if (performance_test) {
176
+ dummy_kernel<<<1, 1, 0, stream>>>();
177
+ constexpr int warmup_iters = 5;
178
+ constexpr int num_iters = 100;
179
+ // warmup
180
+ for (int i = 0; i < warmup_iters; i++) {
181
+ NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
182
+ comm, stream));
183
+ }
184
+ CUDACHECK(cudaEventRecord(start, stream));
185
+ for (int i = 0; i < num_iters; i++) {
186
+ NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
187
+ comm, stream));
188
+ }
189
+ CUDACHECK(cudaEventRecord(stop, stream));
190
+ CUDACHECK(cudaStreamSynchronize(stream));
191
+ float allreduce_ms = 0;
192
+ cudaEventElapsedTime(&allreduce_ms, start, stop);
193
173
- dummy_kernel<<<1, 1, 0, stream>>>();
174
- constexpr int warmup_iters = 5;
175
- constexpr int num_iters = 25;
176
- // warmup
177
- for (int i = 0; i < warmup_iters; i++) {
178
- NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
179
- stream));
180
- }
181
- CUDACHECK(cudaEventRecord(start, stream));
182
- for (int i = 0; i < num_iters; i++) {
183
- NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
184
- stream));
185
- }
186
- CUDACHECK(cudaEventRecord(stop, stream));
187
- CUDACHECK(cudaStreamSynchronize(stream));
188
- float allreduce_ms = 0;
189
- cudaEventElapsedTime(&allreduce_ms, start, stop);
190
-
191
- // if (myRank == 1) dummy_kernel<<<1, 1, 0, stream>>>();
192
- // set_data<T><<<16, 1024, 0, stream>>>(self_data, data_size, myRank);
193
-
194
- dummy_kernel<<<1, 1, 0, stream>>>();
195
- // warm up
196
- for (int i = 0; i < warmup_iters; i++) {
197
- fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
198
- }
199
- CUDACHECK(cudaEventRecord(start, stream));
200
- for (int i = 0; i < num_iters; i++) {
201
- fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
202
- }
203
- CUDACHECK(cudaEventRecord(stop, stream));
204
- CUDACHECK(cudaStreamSynchronize(stream));
205
-
206
- float duration_ms = 0;
207
- cudaEventElapsedTime(&duration_ms, start, stop);
208
- if (myRank == 0)
209
- printf(
210
- "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
211
- "time:%.2fus\n",
212
- myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
213
- duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
194
+ dummy_kernel<<<1, 1, 0, stream>>>();
195
+ // warm up
196
+ for (int i = 0; i < warmup_iters; i++) {
197
+ fa.allreduce<T>(stream, self_data, result, data_size, threads,
198
+ block_limit);
199
+ }
200
+ CUDACHECK(cudaEventRecord(start, stream));
201
+ for (int i = 0; i < num_iters; i++) {
202
+ fa.allreduce<T>(stream, self_data, result, data_size, threads,
203
+ block_limit);
204
+ }
205
+ CUDACHECK(cudaEventRecord(stop, stream));
206
+ CUDACHECK(cudaStreamSynchronize(stream));
207
215
- // And wait for all the queued up work to complete
216
- CUDACHECK(cudaStreamSynchronize(stream));
208
+ float duration_ms = 0;
209
+ cudaEventElapsedTime(&duration_ms, start, stop);
210
+ if (myRank == 0)
211
+ printf(
212
+ "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
213
+ "time:%.2fus\n",
214
+ myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
215
+ duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
216
218
- NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
219
- ncclSum, comm, stream));
217
+ // And wait for all the queued up work to complete
218
+ CUDACHECK(cudaStreamSynchronize(stream));
219
221
- double *nccl_result, *my_result;
222
- CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
223
- CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
220
+ NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
221
+ ncclSum, comm, stream));
222
225
- convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
226
- my_result, data_size);
227
- CUDACHECK(cudaStreamSynchronize(stream));
223
+ convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
224
+ my_result, data_size);
225
+ CUDACHECK(cudaStreamSynchronize(stream));
226
229
- for (unsigned long j = 0; j < data_size; j++) {
230
- auto diff = abs(nccl_result[j] - my_result[j]);
231
- if (diff >= 1e-2) {
232
- printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
233
- myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
234
- break;
227
+ for (unsigned long j = 0; j < data_size; j++) {
228
+ auto diff = abs(nccl_result[j] - my_result[j]);
229
+ if (diff >= 4e-2) {
230
+ printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
231
+ myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
232
+ break;
233
+ }
234
}
236
- }
235
+ long double nccl_diffs = 0.0;
236
+ long double my_diffs = 0.0;
237
+ for (int j = 0; j < data_size; j++) {
238
+ nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
239
+ my_diffs += abs(my_result[j] - ground_truth[j]);
240
+ }
241
+ if (myRank == 0)
242
+ std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
243
+ << " me: " << my_diffs / data_size << std::endl;
244
+ } else {
245
+ for (int i = 0; i < 100; i++) {
246
+ fa.allreduce<T>(stream, self_data, result, data_size, threads,
247
+ block_limit);
248
+ CUDACHECK(cudaStreamSynchronize(stream));
249
+ NCCLCHECK(ncclAllReduce(self_data, self_data_copy, data_size, ncclDtype,
250
+ ncclSum, comm, stream));
251
+ convert_data<T><<<108, 1024, 0, stream>>>(
252
+ self_data_copy, result, nccl_result, my_result, data_size);
253
+ CUDACHECK(cudaStreamSynchronize(stream));
254
238
- long double nccl_diffs = 0.0;
239
- long double my_diffs = 0.0;
240
- for (int j = 0; j < data_size; j++) {
241
- nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
242
- my_diffs += abs(my_result[j] - ground_truth[j]);
255
+ for (unsigned long j = 0; j < data_size; j++) {
256
+ auto diff = abs(nccl_result[j] - my_result[j]);
257
+ if (diff >= 4e-2) {
258
+ printf(
259
+ "Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
260
+ myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
261
+ break;
262
+ }
263
+ }
264
+ }
265
+ if (myRank == 0)
266
+ printf("Test passed: nGPUs:%d, sz (kb): %d, %d, %d\n", nRanks,
267
+ data_size * sizeof(T) / 1024, threads, block_limit);
268
+ // long double nccl_diffs = 0.0;
269
+ // long double my_diffs = 0.0;
270
+ // for (int j = 0; j < data_size; j++) {
271
+ // nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
272
+ // my_diffs += abs(my_result[j] - ground_truth[j]);
273
+ // }
274
+ // if (myRank == 0)
275
+ // std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
276
+ // << " me: " << my_diffs / data_size << std::endl;
277
}
244
- if (myRank == 0)
245
- std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
246
- << " me: " << my_diffs / data_size << std::endl;
278
279
CUDACHECK(cudaFree(result));
280
CUDACHECK(cudaFree(self_data_copy));
@@ -269,14 +300,15 @@ int main(int argc, char **argv) {
300
MPI_COMM_WORLD));
301
NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
302
303
+ bool performance_test = true;
304
cudaProfilerStart();
305
// for (int threads : {256, 512}) {
306
// for (int block_limit = 16; block_limit < 112; block_limit += 4) {
307
// run<half>(myRank, nRanks, comm, threads, block_limit, 4096 * 1024);
308
// }
309
// }
278
- for (int sz = 512; sz <= (32 << 20); sz *= 2) {
279
- run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 50);
310
+ for (int sz = 512; sz <= (8 << 20); sz *= 2) {
311
+ run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 47, performance_test);
312
}
313
314
cudaProfilerStop();
vllm/config.py
-9
@@ -506,15 +506,6 @@ class ParallelConfig:
506
raise ValueError("Unable to use nsight profiling unless workers "
507
"run with Ray.")
508
509
- # FIXME(woosuk): Fix the stability issues and re-enable the custom
510
- # all-reduce kernel.
511
- if not self.disable_custom_all_reduce and self.world_size > 1:
512
- self.disable_custom_all_reduce = True
513
- logger.info(
514
- "Custom all-reduce kernels are temporarily disabled due to "
515
- "stability issues. We will re-enable them once the issues are "
516
- "resolved.")
517
-
509
510
class SchedulerConfig:
511
"""Scheduler configuration.
vllm/entrypoints/llm.py
+1
-1
@@ -83,7 +83,7 @@ class LLM:
83
swap_space: int = 4,
84
enforce_eager: bool = False,
85
max_context_len_to_capture: int = 8192,
86
- disable_custom_all_reduce: bool = False,
86
+ disable_custom_all_reduce: bool = True,
87
**kwargs,
88
) -> None:
89
if "disable_log_stats" not in kwargs:
vllm/model_executor/parallel_utils/custom_all_reduce.py
+43
-7
@@ -37,16 +37,23 @@ def init_custom_ar() -> None:
37
logger.warn(
38
"Custom allreduce is disabled due to an unsupported world size: "
39
"%d. Supported world sizes: %s. To silence this warning, specify"
40
- "disable_custom_all_reduce=True explicitly.", world_size,
40
+ " disable_custom_all_reduce=True explicitly.", world_size,
41
str(_SUPPORTED_WORLD_SIZES))
42
return
43
if not _can_p2p(rank, world_size):
44
logger.warn(
45
"Custom allreduce is disabled because your platform lacks GPU P2P"
46
- " capability. To silence this warning, specify"
47
- "disable_custom_all_reduce=True explicitly.")
46
+ " capability or P2P test failed. To silence this warning, specify"
47
+ " disable_custom_all_reduce=True explicitly.")
48
return
49
- _CA_HANDLE = CustomAllreduce(rank, world_size)
49
+ full_nvlink = _is_full_nvlink(rank, world_size)
50
+ if world_size > 2 and not full_nvlink:
51
+ logger.warn(
52
+ "Custom allreduce is disabled because it's not supported on more"
53
+ " than two PCIe-only GPUs. To silence this warning, specify"
54
+ " disable_custom_all_reduce=True explicitly.")
55
+ return
56
+ _CA_HANDLE = CustomAllreduce(rank, world_size, full_nvlink)
57
58
59
def begin_capture() -> None:
@@ -134,18 +141,48 @@ def _is_full_nvlink(rank, world_size):
141
142
143
def _can_p2p(rank: int, world_size: int) -> bool:
144
+ num_dev = torch.cuda.device_count()
145
+ # note: num dev can be larger than world_size if we're only using
146
+ # first few GPUs
147
+ if num_dev < world_size:
148
+ logger.warn(
149
+ "Cannot test GPU P2P because not all GPUs are visible to the "
150
+ "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
151
+ " is set.")
152
+ return False
153
for i in range(world_size):
154
if i == rank:
155
continue
156
if not torch.cuda.can_device_access_peer(rank, i):
157
return False
158
+ # on some platforms, P2P support might be buggy and we need
159
+ # additional checks. See also:
160
+ # https://github.com/vllm-project/vllm/issues/2728
161
+ if not _can_actually_p2p(rank, i):
162
+ return False
163
return True
164
165
166
+# code partly borrowed from
167
+# https://github.com/turboderp/exllamav2/blob/1c67f97f3d2a968605a9c31ab791a05c85bb7879/exllamav2/compat.py#L10
168
+# License: MIT
169
+def _can_actually_p2p(idx_a, idx_b):
170
+ dev_i = f"cuda:{idx_a}"
171
+ dev_j = f"cuda:{idx_b}"
172
+ a = torch.randn(5, device=dev_i) + 123.0
173
+ b = a.to(dev_j)
174
+ c = b.to(dev_i)
175
+ return torch.all(a == c)
176
+
177
+
178
class CustomAllreduce:
179
180
# max_size: max supported allreduce size
148
- def __init__(self, rank, world_size, max_size=8192 * 1024) -> None:
181
+ def __init__(self,
182
+ rank,
183
+ world_size,
184
+ full_nvlink,
185
+ max_size=8192 * 1024) -> None:
186
# buffers memory are owned by this Python class and passed to C++
187
# meta data composes of two parts: meta data for synchronization
188
# (256 bytes) and a temporary buffer for storing intermediate
@@ -167,11 +204,10 @@ class CustomAllreduce:
204
self.max_size = max_size
205
self.world_size = world_size
206
handles, offsets = self._get_ipc_meta(self.meta)
170
- self.full_nvlink = _is_full_nvlink(rank, world_size)
207
+ self.full_nvlink = full_nvlink
208
self._ptr = custom_ar.init_custom_ar(self.meta, self.rank_data,
209
handles, offsets, rank,
210
self.full_nvlink)
174
- self.fast_cond = self.full_nvlink or world_size <= 2
211
self.register_buffer(self.buffer)
212
213
def _get_ipc_meta(self, inp: torch.Tensor):