@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f721096d48
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[BugFix] Some fixes for custom allreduce kernels (#2760)

Hanzhi Zhou committed Mar 21, 2024 at 23:02 UTC f721096d48a7e3b98dffcb9b400bf58989cef64d
6 files changed +232 -250
csrc/custom_all_reduce.cu
+5 -5
@@ -29,7 +29,7 @@ fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
29 std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
30 }
31 return (fptr_t) new vllm::CustomAllreduce(
32 - reinterpret_cast<vllm::Metadata *>(meta.data_ptr()), rank_data.data_ptr(),
32 + reinterpret_cast<vllm::Signal *>(meta.data_ptr()), rank_data.data_ptr(),
33 rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
34 }
35
@@ -62,9 +62,9 @@ bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
62 if (inp_size % 16 != 0) return false;
63 if (!_is_weak_contiguous(inp)) return false;
64 if (world_size == 2 || full_nvlink) return inp_size <= max_size;
65 - // 4 PCIE GPUs use 2 stage allreduce, and is only faster than NCCL when size
66 - // <= 512k
67 - return world_size <= 4 && inp_size <= 512 * 1024;
65 + // for 4 or more non NVLink-capable GPUs, custom allreduce provides little
66 + // performance improvement over NCCL.
67 + return false;
68 }
69
70 void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
@@ -126,7 +126,7 @@ void dispose(fptr_t _fa) {
126 delete fa;
127 }
128
129 -int meta_size() { return sizeof(vllm::Metadata); }
129 +int meta_size() { return sizeof(vllm::Signal); }
130
131 void register_buffer(fptr_t _fa, torch::Tensor &t,
132 const std::vector<std::string> &handles,
csrc/custom_all_reduce.cuh
+75 -152
@@ -23,29 +23,17 @@
23
24 namespace vllm {
25
26 +constexpr int kMaxBlocks = 64;
27 +// note: we don't want to use atomics for signals because peer atomics are no
28 +// supported on PCIe links
29 struct Signal {
27 - alignas(64) union {
28 - uint64_t flag;
29 - unsigned char data[8];
30 - } start;
31 - alignas(64) union {
32 - uint64_t flag;
33 - unsigned char data[8];
34 - } end;
30 + alignas(128) uint32_t start[kMaxBlocks][8];
31 + alignas(128) uint32_t end[kMaxBlocks][8];
32 };
33
37 -struct Metadata {
38 - alignas(128) Signal sg;
39 - alignas(128) int counter;
40 -};
41 -static_assert(offsetof(Metadata, counter) == 128);
42 -static_assert(sizeof(Metadata) == 256);
43 -
34 struct __align__(16) RankData { const void *__restrict__ ptrs[8]; };
35
46 -struct RankSignals {
47 - volatile Signal *signals[8];
48 -};
36 +struct __align__(16) RankSignals { volatile Signal *signals[8]; };
37
38 // like std::array, but aligned
39 template <typename T, int sz>
@@ -135,70 +123,49 @@ DINLINE O downcast(array_t<float, O::size> val) {
123 }
124 }
125
138 -// compute flag at compile time
139 -__host__ __device__ constexpr uint64_t compute_flag(int ngpus) {
140 - auto m = std::numeric_limits<uint64_t>::max();
141 - return m >> ((8 - ngpus) * 8);
142 -}
143 -
126 +// This function is meant to be used as the first synchronization in the all
127 +// reduce kernel. Thus, it doesn't need to make any visibility guarantees for
128 +// prior memory accesses. Note: volatile writes will not be reordered against
129 +// other volatile writes.
130 template <int ngpus>
145 -DINLINE void start_sync(const RankSignals &sg, volatile Metadata *meta,
131 +DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
132 int rank) {
147 - constexpr auto FLAG = compute_flag(ngpus);
148 - if (blockIdx.x == 0) {
149 - if (threadIdx.x < ngpus)
150 - // simultaneously write to the corresponding byte to all other ranks.
151 - // Latency = 1 p2p write
152 - sg.signals[threadIdx.x]->start.data[rank] = 255;
153 - else if (threadIdx.x == 32)
154 - // reset
155 - meta->sg.end.flag = 0;
156 - }
157 - if (threadIdx.x == 0) {
158 - while (meta->sg.start.flag != FLAG)
133 + if (threadIdx.x < ngpus) {
134 + // reset flag for next time
135 + self_sg->end[blockIdx.x][threadIdx.x] = 0;
136 + // simultaneously write to the corresponding flag of all ranks.
137 + // Latency = 1 p2p write
138 + sg.signals[threadIdx.x]->start[blockIdx.x][rank] = 1;
139 + // wait until we got true from all ranks
140 + while (!self_sg->start[blockIdx.x][threadIdx.x])
141 ;
142 }
143 __syncthreads();
144 }
145
146 +// This function is meant to be used as the second or the final synchronization
147 +// barrier in the all reduce kernel. If it's the final synchronization barrier,
148 +// we don't need to make any visibility guarantees for prior memory accesses.
149 template <int ngpus, bool final_sync = false>
165 -DINLINE void end_sync(const RankSignals &sg, volatile Metadata *meta,
150 +DINLINE void end_sync(const RankSignals &sg, volatile Signal *self_sg,
151 int rank) {
167 - constexpr auto FLAG = compute_flag(ngpus);
152 __syncthreads();
169 - __shared__ int num;
170 - if (threadIdx.x == 0) num = atomicAdd((int *)&meta->counter, 1);
171 - __syncthreads();
172 -
173 - // Only the last completing block can perform the end synchronization
174 - // This can ensures when the final busy wait ends, all ranks must have
175 - // finished reading each other's buffer.
176 - if (num == gridDim.x - 1) {
177 - if (threadIdx.x == 32) {
178 - // reset in a different warp
179 - meta->counter = 0;
180 - meta->sg.start.flag = 0;
181 - } else if (threadIdx.x < ngpus) {
182 - // simultaneously write to the corresponding byte to all other ranks.
183 - // Latency = 1 p2p write
184 - sg.signals[threadIdx.x]->end.data[rank] = 255;
185 - }
186 - // if this is the final sync, only one block needs it
187 - // because kernel exit can serve as sync
188 - if constexpr (final_sync) {
189 - if (threadIdx.x == 0) {
190 - while (meta->sg.end.flag != FLAG)
191 - ;
192 - }
193 - }
194 - }
195 - if constexpr (!final_sync) {
196 - if (threadIdx.x == 0) {
197 - while (meta->sg.end.flag != FLAG)
198 - ;
199 - }
200 - __syncthreads();
153 + // eliminate the case that prior writes are not visible after signals become
154 + // visible. Note that I did not managed to make this happen through a lot of
155 + // testing. Might be the case that hardware provides stronger guarantee than
156 + // the memory model.
157 + if constexpr (!final_sync) __threadfence_system();
158 + if (threadIdx.x < ngpus) {
159 + // reset flag for next time
160 + self_sg->start[blockIdx.x][threadIdx.x] = 0;
161 + // simultaneously write to the corresponding flag of all ranks.
162 + // Latency = 1 p2p write
163 + sg.signals[threadIdx.x]->end[blockIdx.x][rank] = 1;
164 + // wait until we got true from all ranks
165 + while (!self_sg->end[blockIdx.x][threadIdx.x])
166 + ;
167 }
168 + if constexpr (!final_sync) __syncthreads();
169 }
170
171 template <typename P, int ngpus, typename A>
@@ -214,32 +181,32 @@ DINLINE P packed_reduce(const P *ptrs[], int idx) {
181 template <typename T, int ngpus>
182 __global__ void __launch_bounds__(512, 1)
183 cross_device_reduce_1stage(RankData *_dp, RankSignals sg,
217 - volatile Metadata *meta, T *__restrict__ result,
184 + volatile Signal *self_sg, T *__restrict__ result,
185 int rank, int size) {
186 using P = typename packed_t<T>::P;
187 using A = typename packed_t<T>::A;
188 // note: we don't reorder the address so the accumulation order is the same
189 // for all ranks, ensuring bitwise identical results
190 auto dp = *_dp;
224 - start_sync<ngpus>(sg, meta, rank);
191 + start_sync<ngpus>(sg, self_sg, rank);
192 // do the actual reduction
193 for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
194 idx += gridDim.x * blockDim.x) {
195 ((P *)result)[idx] =
196 packed_reduce<P, ngpus, A>((const P **)&dp.ptrs[0], idx);
197 }
231 - end_sync<ngpus, true>(sg, meta, rank);
198 + end_sync<ngpus, true>(sg, self_sg, rank);
199 }
200
201 template <typename P>
202 DINLINE P *get_tmp_buf(volatile Signal *sg) {
236 - return (P *)(((Metadata *)sg) + 1);
203 + return (P *)(((Signal *)sg) + 1);
204 }
205
206 template <typename T, int ngpus>
207 __global__ void __launch_bounds__(512, 1)
208 cross_device_reduce_2stage(RankData *_dp, RankSignals sg,
242 - volatile Metadata *meta, T *__restrict__ result,
209 + volatile Signal *self_sg, T *__restrict__ result,
210 int rank, int size) {
211 int tid = blockIdx.x * blockDim.x + threadIdx.x;
212 int stride = gridDim.x * blockDim.x;
@@ -248,6 +215,7 @@ __global__ void __launch_bounds__(512, 1)
215 int part = size / ngpus;
216 int start = rank * part;
217 int end = rank == ngpus - 1 ? size : start + part;
218 + int largest_part = part + size % ngpus;
219 const P *ptrs[ngpus];
220 P *tmps[ngpus];
221 #pragma unroll
@@ -257,75 +225,28 @@ __global__ void __launch_bounds__(512, 1)
225 tmps[i] = get_tmp_buf<P>(sg.signals[target]);
226 }
227 auto tmp_out = tmps[0];
260 - start_sync<ngpus>(sg, meta, rank);
228 + start_sync<ngpus>(sg, self_sg, rank);
229 // stage 1: reduce scatter
230 for (int idx = start + tid; idx < end; idx += stride) {
231 tmp_out[idx - start] = packed_reduce<P, ngpus, A>(ptrs, idx);
232 }
265 - // Maybe TODO: replace this with per-block release-acquire
266 - // can save about 1-2us (not a lot though)
267 - end_sync<ngpus>(sg, meta, rank);
268 -
269 - // stage 2: allgather
270 - for (int idx = tid; idx < part; idx += stride) {
233 + end_sync<ngpus>(sg, self_sg, rank);
234 +
235 + // stage 2: allgather. Note: it's important to match the tid between
236 + // the two stages, because visibility across devices is only guaranteed
237 + // between threads that have the same tid. If thread i computes the sum of
238 + // start + i in the first stage, then thread i also gathers start + i from all
239 + // ranks.
240 + for (int idx = tid; idx < largest_part; idx += stride) {
241 #pragma unroll
242 for (int i = 0; i < ngpus; i++) {
273 - int dst_idx = ((rank + i) % ngpus) * part + idx;
274 - ((P *)result)[dst_idx] = tmps[i][idx];
275 - }
276 - }
277 - // process the last larger partition
278 - int remaining = size - part * ngpus;
279 - if (tid < remaining) {
280 - int dst_idx = tid + part * ngpus;
281 - ((P *)result)[dst_idx] = get_tmp_buf<P>(sg.signals[ngpus - 1])[part + tid];
282 - }
283 -
284 - // faster than this
285 - // for (int idx = tid; idx < size; idx += stride) {
286 - // int target_rank = idx / part;
287 - // if (target_rank == ngpus) target_rank -= 1;
288 - // ((P *)result)[idx] = tmps[target_rank][idx - target_rank * part];
289 - // }
290 -}
291 -
292 -template <typename T, int ngpus>
293 -__global__ void __launch_bounds__(512, 1)
294 - cross_device_reduce_half_butterfly(RankData *_dp, RankSignals sg,
295 - volatile Metadata *meta,
296 - T *__restrict__ result, int rank,
297 - int size) {
298 - int tid = blockIdx.x * blockDim.x + threadIdx.x;
299 - int stride = gridDim.x * blockDim.x;
300 - using P = typename packed_t<T>::P;
301 - using A = typename packed_t<T>::A;
302 - auto tmp_out = get_tmp_buf<P>(sg.signals[rank]);
303 - constexpr int hg = ngpus / 2;
304 - // Actually not quite half butterfly.
305 - // This is an all-to-all within each group containing half of the ranks
306 - // followed by cross-group add. Equivalent to half butterfly when there
307 - // are 4 GPUs, a common case for PCIe cards like T4 and A10.
308 - const P *ptrs[hg];
309 - {
310 - int start = rank - rank % hg;
311 -#pragma unroll
312 - for (int i = 0; i < hg; i++) {
313 - ptrs[i] = (const P *)_dp->ptrs[i + start];
243 + int gather_from_rank = ((rank + i) % ngpus);
244 + if (gather_from_rank == ngpus - 1 || idx < part) {
245 + int dst_idx = gather_from_rank * part + idx;
246 + ((P *)result)[dst_idx] = tmps[i][idx];
247 + }
248 }
249 }
316 - start_sync<ngpus>(sg, meta, rank);
317 - for (int idx = tid; idx < size; idx += stride) {
318 - tmp_out[idx] = packed_reduce<P, hg, A>(ptrs, idx);
319 - }
320 - end_sync<ngpus>(sg, meta, rank);
321 -
322 - auto src = get_tmp_buf<P>(sg.signals[(ngpus - 1) - rank % ngpus]);
323 - // do the cross group reduction
324 - for (int idx = tid; idx < size; idx += stride) {
325 - auto tmp = tmp_out[idx];
326 - packed_assign_add(tmp, src[idx]);
327 - ((P *)result)[idx] = tmp;
328 - }
250 }
251
252 using IPC_KEY = std::array<uint8_t, sizeof(cudaIpcMemHandle_t)>;
@@ -341,7 +262,7 @@ class CustomAllreduce {
262 // below are device pointers
263 RankSignals sg_;
264 std::unordered_map<void *, RankData *> buffers_;
344 - Metadata *meta_;
265 + Signal *self_sg_;
266
267 // stores the registered device pointers from all ranks
268 RankData *d_rank_data_base_, *d_rank_data_end_;
@@ -352,32 +273,32 @@ class CustomAllreduce {
273 /**
274 * meta is a pointer to device metadata and temporary buffer for allreduce.
275 *
355 - * There's a total of sizeof(Metadata) of prefix before the actual data,
276 + * There's a total of sizeof(Signal) of prefix before the actual data,
277 * so meta + 1 points to actual temporary buffer.
278 *
279 * note: this class does not own any device memory. Any required buffers
280 * are passed in from the constructor
281 */
361 - CustomAllreduce(Metadata *meta, void *rank_data, size_t rank_data_sz,
282 + CustomAllreduce(Signal *meta, void *rank_data, size_t rank_data_sz,
283 const cudaIpcMemHandle_t *handles,
284 const std::vector<int64_t> &offsets, int rank,
285 bool full_nvlink = true)
286 : rank_(rank),
287 world_size_(offsets.size()),
288 full_nvlink_(full_nvlink),
368 - meta_(meta),
289 + self_sg_(meta),
290 d_rank_data_base_(reinterpret_cast<RankData *>(rank_data)),
291 d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
292 for (int i = 0; i < world_size_; i++) {
372 - Metadata *rank_meta;
293 + Signal *rank_sg;
294 if (i != rank_) {
295 char *handle = open_ipc_handle(&handles[i]);
296 handle += offsets[i];
376 - rank_meta = (Metadata *)handle;
297 + rank_sg = (Signal *)handle;
298 } else {
378 - rank_meta = meta_;
299 + rank_sg = self_sg_;
300 }
380 - sg_.signals[i] = &rank_meta->sg;
301 + sg_.signals[i] = rank_sg;
302 }
303 }
304
@@ -492,6 +413,10 @@ class CustomAllreduce {
413 "custom allreduce currently requires input length to be multiple "
414 "of " +
415 std::to_string(d));
416 + if (block_limit > kMaxBlocks)
417 + throw std::runtime_error("max supported block limit is " +
418 + std::to_string(kMaxBlocks) + ". Got " +
419 + std::to_string(block_limit));
420
421 RankData *ptrs;
422 cudaStreamCaptureStatus status;
@@ -512,9 +437,9 @@ class CustomAllreduce {
437 size /= d;
438 auto bytes = size * sizeof(typename packed_t<T>::P);
439 int blocks = std::min(block_limit, (size + threads - 1) / threads);
515 -#define KL(ngpus, name) \
516 - name<T, ngpus> \
517 - <<<blocks, threads, 0, stream>>>(ptrs, sg_, meta_, output, rank_, size);
440 +#define KL(ngpus, name) \
441 + name<T, ngpus><<<blocks, threads, 0, stream>>>(ptrs, sg_, self_sg_, output, \
442 + rank_, size);
443 #define REDUCE_CASE(ngpus) \
444 case ngpus: { \
445 if (world_size_ == 2) { \
@@ -526,8 +451,6 @@ class CustomAllreduce {
451 } else { \
452 KL(ngpus, cross_device_reduce_2stage); \
453 } \
529 - } else { \
530 - KL(ngpus, cross_device_reduce_half_butterfly); \
454 } \
455 break; \
456 }
@@ -556,7 +479,7 @@ class CustomAllreduce {
479 /**
480 * To inspect PTX/SASS, copy paste this header file to compiler explorer and add
481 a template instantiation:
559 - * template void CustomAllreduce::allreduce<half>(cudaStream_t, half *, half *,
560 - int, int, int);
482 + * template void vllm::CustomAllreduce::allreduce<half>(cudaStream_t, half *,
483 + half *, int, int, int);
484 */
485 } // namespace vllm
csrc/custom_all_reduce_test.cu
+108 -76
@@ -92,7 +92,7 @@ __global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
92
93 template <typename T>
94 void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
95 - int data_size) {
95 + int data_size, bool performance_test) {
96 T *result;
97 cudaStream_t stream;
98 CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
@@ -101,7 +101,7 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
101
102 cudaIpcMemHandle_t self_data_handle;
103 cudaIpcMemHandle_t data_handles[8];
104 - vllm::Metadata *buffer;
104 + vllm::Signal *buffer;
105 T *self_data_copy;
106 /**
107 * Allocate IPC buffer
@@ -115,9 +115,9 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
115 * convenience.
116 */
117 CUDACHECK(
118 - cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
119 - CUDACHECK(cudaMemset(buffer, 0,
120 - 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
118 + cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
119 + CUDACHECK(
120 + cudaMemset(buffer, 0, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
121 CUDACHECK(cudaMalloc(&self_data_copy, data_size * sizeof(T)));
122 CUDACHECK(cudaIpcGetMemHandle(&self_data_handle, buffer));
123
@@ -133,7 +133,7 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
133 offsets, myRank);
134 auto *self_data =
135 reinterpret_cast<T *>(reinterpret_cast<char *>(buffer) +
136 - sizeof(vllm::Metadata) + data_size * sizeof(T));
136 + sizeof(vllm::Signal) + data_size * sizeof(T));
137 // hack buffer registration
138 {
139 std::vector<std::string> handles;
@@ -143,8 +143,8 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
143 char *end = (char *)&data_handles[i + 1];
144 handles.emplace_back(begin, end);
145 }
146 - std::vector<int64_t> offsets(
147 - nRanks, sizeof(vllm::Metadata) + data_size * sizeof(T));
146 + std::vector<int64_t> offsets(nRanks,
147 + sizeof(vllm::Signal) + data_size * sizeof(T));
148 fa.register_buffer(handles, offsets, self_data);
149 }
150
@@ -169,81 +169,112 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
169 } else {
170 ncclDtype = ncclFloat;
171 }
172 + double *nccl_result, *my_result;
173 + CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
174 + CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
175 + if (performance_test) {
176 + dummy_kernel<<<1, 1, 0, stream>>>();
177 + constexpr int warmup_iters = 5;
178 + constexpr int num_iters = 100;
179 + // warmup
180 + for (int i = 0; i < warmup_iters; i++) {
181 + NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
182 + comm, stream));
183 + }
184 + CUDACHECK(cudaEventRecord(start, stream));
185 + for (int i = 0; i < num_iters; i++) {
186 + NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
187 + comm, stream));
188 + }
189 + CUDACHECK(cudaEventRecord(stop, stream));
190 + CUDACHECK(cudaStreamSynchronize(stream));
191 + float allreduce_ms = 0;
192 + cudaEventElapsedTime(&allreduce_ms, start, stop);
193
173 - dummy_kernel<<<1, 1, 0, stream>>>();
174 - constexpr int warmup_iters = 5;
175 - constexpr int num_iters = 25;
176 - // warmup
177 - for (int i = 0; i < warmup_iters; i++) {
178 - NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
179 - stream));
180 - }
181 - CUDACHECK(cudaEventRecord(start, stream));
182 - for (int i = 0; i < num_iters; i++) {
183 - NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
184 - stream));
185 - }
186 - CUDACHECK(cudaEventRecord(stop, stream));
187 - CUDACHECK(cudaStreamSynchronize(stream));
188 - float allreduce_ms = 0;
189 - cudaEventElapsedTime(&allreduce_ms, start, stop);
190 -
191 - // if (myRank == 1) dummy_kernel<<<1, 1, 0, stream>>>();
192 - // set_data<T><<<16, 1024, 0, stream>>>(self_data, data_size, myRank);
193 -
194 - dummy_kernel<<<1, 1, 0, stream>>>();
195 - // warm up
196 - for (int i = 0; i < warmup_iters; i++) {
197 - fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
198 - }
199 - CUDACHECK(cudaEventRecord(start, stream));
200 - for (int i = 0; i < num_iters; i++) {
201 - fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
202 - }
203 - CUDACHECK(cudaEventRecord(stop, stream));
204 - CUDACHECK(cudaStreamSynchronize(stream));
205 -
206 - float duration_ms = 0;
207 - cudaEventElapsedTime(&duration_ms, start, stop);
208 - if (myRank == 0)
209 - printf(
210 - "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
211 - "time:%.2fus\n",
212 - myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
213 - duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
194 + dummy_kernel<<<1, 1, 0, stream>>>();
195 + // warm up
196 + for (int i = 0; i < warmup_iters; i++) {
197 + fa.allreduce<T>(stream, self_data, result, data_size, threads,
198 + block_limit);
199 + }
200 + CUDACHECK(cudaEventRecord(start, stream));
201 + for (int i = 0; i < num_iters; i++) {
202 + fa.allreduce<T>(stream, self_data, result, data_size, threads,
203 + block_limit);
204 + }
205 + CUDACHECK(cudaEventRecord(stop, stream));
206 + CUDACHECK(cudaStreamSynchronize(stream));
207
215 - // And wait for all the queued up work to complete
216 - CUDACHECK(cudaStreamSynchronize(stream));
208 + float duration_ms = 0;
209 + cudaEventElapsedTime(&duration_ms, start, stop);
210 + if (myRank == 0)
211 + printf(
212 + "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
213 + "time:%.2fus\n",
214 + myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
215 + duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
216
218 - NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
219 - ncclSum, comm, stream));
217 + // And wait for all the queued up work to complete
218 + CUDACHECK(cudaStreamSynchronize(stream));
219
221 - double *nccl_result, *my_result;
222 - CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
223 - CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
220 + NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
221 + ncclSum, comm, stream));
222
225 - convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
226 - my_result, data_size);
227 - CUDACHECK(cudaStreamSynchronize(stream));
223 + convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
224 + my_result, data_size);
225 + CUDACHECK(cudaStreamSynchronize(stream));
226
229 - for (unsigned long j = 0; j < data_size; j++) {
230 - auto diff = abs(nccl_result[j] - my_result[j]);
231 - if (diff >= 1e-2) {
232 - printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
233 - myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
234 - break;
227 + for (unsigned long j = 0; j < data_size; j++) {
228 + auto diff = abs(nccl_result[j] - my_result[j]);
229 + if (diff >= 4e-2) {
230 + printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
231 + myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
232 + break;
233 + }
234 }
236 - }
235 + long double nccl_diffs = 0.0;
236 + long double my_diffs = 0.0;
237 + for (int j = 0; j < data_size; j++) {
238 + nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
239 + my_diffs += abs(my_result[j] - ground_truth[j]);
240 + }
241 + if (myRank == 0)
242 + std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
243 + << " me: " << my_diffs / data_size << std::endl;
244 + } else {
245 + for (int i = 0; i < 100; i++) {
246 + fa.allreduce<T>(stream, self_data, result, data_size, threads,
247 + block_limit);
248 + CUDACHECK(cudaStreamSynchronize(stream));
249 + NCCLCHECK(ncclAllReduce(self_data, self_data_copy, data_size, ncclDtype,
250 + ncclSum, comm, stream));
251 + convert_data<T><<<108, 1024, 0, stream>>>(
252 + self_data_copy, result, nccl_result, my_result, data_size);
253 + CUDACHECK(cudaStreamSynchronize(stream));
254
238 - long double nccl_diffs = 0.0;
239 - long double my_diffs = 0.0;
240 - for (int j = 0; j < data_size; j++) {
241 - nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
242 - my_diffs += abs(my_result[j] - ground_truth[j]);
255 + for (unsigned long j = 0; j < data_size; j++) {
256 + auto diff = abs(nccl_result[j] - my_result[j]);
257 + if (diff >= 4e-2) {
258 + printf(
259 + "Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
260 + myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
261 + break;
262 + }
263 + }
264 + }
265 + if (myRank == 0)
266 + printf("Test passed: nGPUs:%d, sz (kb): %d, %d, %d\n", nRanks,
267 + data_size * sizeof(T) / 1024, threads, block_limit);
268 + // long double nccl_diffs = 0.0;
269 + // long double my_diffs = 0.0;
270 + // for (int j = 0; j < data_size; j++) {
271 + // nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
272 + // my_diffs += abs(my_result[j] - ground_truth[j]);
273 + // }
274 + // if (myRank == 0)
275 + // std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
276 + // << " me: " << my_diffs / data_size << std::endl;
277 }
244 - if (myRank == 0)
245 - std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
246 - << " me: " << my_diffs / data_size << std::endl;
278
279 CUDACHECK(cudaFree(result));
280 CUDACHECK(cudaFree(self_data_copy));
@@ -269,14 +300,15 @@ int main(int argc, char **argv) {
300 MPI_COMM_WORLD));
301 NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
302
303 + bool performance_test = true;
304 cudaProfilerStart();
305 // for (int threads : {256, 512}) {
306 // for (int block_limit = 16; block_limit < 112; block_limit += 4) {
307 // run<half>(myRank, nRanks, comm, threads, block_limit, 4096 * 1024);
308 // }
309 // }
278 - for (int sz = 512; sz <= (32 << 20); sz *= 2) {
279 - run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 50);
310 + for (int sz = 512; sz <= (8 << 20); sz *= 2) {
311 + run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 47, performance_test);
312 }
313
314 cudaProfilerStop();
vllm/config.py
-9
@@ -506,15 +506,6 @@ class ParallelConfig:
506 raise ValueError("Unable to use nsight profiling unless workers "
507 "run with Ray.")
508
509 - # FIXME(woosuk): Fix the stability issues and re-enable the custom
510 - # all-reduce kernel.
511 - if not self.disable_custom_all_reduce and self.world_size > 1:
512 - self.disable_custom_all_reduce = True
513 - logger.info(
514 - "Custom all-reduce kernels are temporarily disabled due to "
515 - "stability issues. We will re-enable them once the issues are "
516 - "resolved.")
517 -
509
510 class SchedulerConfig:
511 """Scheduler configuration.
vllm/entrypoints/llm.py
+1 -1
@@ -83,7 +83,7 @@ class LLM:
83 swap_space: int = 4,
84 enforce_eager: bool = False,
85 max_context_len_to_capture: int = 8192,
86 - disable_custom_all_reduce: bool = False,
86 + disable_custom_all_reduce: bool = True,
87 **kwargs,
88 ) -> None:
89 if "disable_log_stats" not in kwargs:
vllm/model_executor/parallel_utils/custom_all_reduce.py
+43 -7
@@ -37,16 +37,23 @@ def init_custom_ar() -> None:
37 logger.warn(
38 "Custom allreduce is disabled due to an unsupported world size: "
39 "%d. Supported world sizes: %s. To silence this warning, specify"
40 - "disable_custom_all_reduce=True explicitly.", world_size,
40 + " disable_custom_all_reduce=True explicitly.", world_size,
41 str(_SUPPORTED_WORLD_SIZES))
42 return
43 if not _can_p2p(rank, world_size):
44 logger.warn(
45 "Custom allreduce is disabled because your platform lacks GPU P2P"
46 - " capability. To silence this warning, specify"
47 - "disable_custom_all_reduce=True explicitly.")
46 + " capability or P2P test failed. To silence this warning, specify"
47 + " disable_custom_all_reduce=True explicitly.")
48 return
49 - _CA_HANDLE = CustomAllreduce(rank, world_size)
49 + full_nvlink = _is_full_nvlink(rank, world_size)
50 + if world_size > 2 and not full_nvlink:
51 + logger.warn(
52 + "Custom allreduce is disabled because it's not supported on more"
53 + " than two PCIe-only GPUs. To silence this warning, specify"
54 + " disable_custom_all_reduce=True explicitly.")
55 + return
56 + _CA_HANDLE = CustomAllreduce(rank, world_size, full_nvlink)
57
58
59 def begin_capture() -> None:
@@ -134,18 +141,48 @@ def _is_full_nvlink(rank, world_size):
141
142
143 def _can_p2p(rank: int, world_size: int) -> bool:
144 + num_dev = torch.cuda.device_count()
145 + # note: num dev can be larger than world_size if we're only using
146 + # first few GPUs
147 + if num_dev < world_size:
148 + logger.warn(
149 + "Cannot test GPU P2P because not all GPUs are visible to the "
150 + "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
151 + " is set.")
152 + return False
153 for i in range(world_size):
154 if i == rank:
155 continue
156 if not torch.cuda.can_device_access_peer(rank, i):
157 return False
158 + # on some platforms, P2P support might be buggy and we need
159 + # additional checks. See also:
160 + # https://github.com/vllm-project/vllm/issues/2728
161 + if not _can_actually_p2p(rank, i):
162 + return False
163 return True
164
165
166 +# code partly borrowed from
167 +# https://github.com/turboderp/exllamav2/blob/1c67f97f3d2a968605a9c31ab791a05c85bb7879/exllamav2/compat.py#L10
168 +# License: MIT
169 +def _can_actually_p2p(idx_a, idx_b):
170 + dev_i = f"cuda:{idx_a}"
171 + dev_j = f"cuda:{idx_b}"
172 + a = torch.randn(5, device=dev_i) + 123.0
173 + b = a.to(dev_j)
174 + c = b.to(dev_i)
175 + return torch.all(a == c)
176 +
177 +
178 class CustomAllreduce:
179
180 # max_size: max supported allreduce size
148 - def __init__(self, rank, world_size, max_size=8192 * 1024) -> None:
181 + def __init__(self,
182 + rank,
183 + world_size,
184 + full_nvlink,
185 + max_size=8192 * 1024) -> None:
186 # buffers memory are owned by this Python class and passed to C++
187 # meta data composes of two parts: meta data for synchronization
188 # (256 bytes) and a temporary buffer for storing intermediate
@@ -167,11 +204,10 @@ class CustomAllreduce:
204 self.max_size = max_size
205 self.world_size = world_size
206 handles, offsets = self._get_ipc_meta(self.meta)
170 - self.full_nvlink = _is_full_nvlink(rank, world_size)
207 + self.full_nvlink = full_nvlink
208 self._ptr = custom_ar.init_custom_ar(self.meta, self.rank_data,
209 handles, offsets, rank,
210 self.full_nvlink)
174 - self.fast_cond = self.full_nvlink or world_size <= 2
211 self.register_buffer(self.buffer)
212
213 def _get_ipc_meta(self, inp: torch.Tensor):