A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Fix potentially unsafe custom allreduce synchronization (#8558)
Hanzhi Zhou committed
Sep 24, 2024 at 01:08 UTC
cc4325b66ac49e403ed9e1a8c38156a5324e1174
2 files changed
+83
-59
csrc/custom_all_reduce.cuh
+74
-54
@@ -6,6 +6,7 @@
6
#include <cuda_runtime.h>
7
8
#include <iostream>
9
+#include <array>
10
#include <limits>
11
#include <map>
12
#include <unordered_map>
@@ -23,17 +24,23 @@
24
25
namespace vllm {
26
26
-constexpr int kMaxBlocks = 64;
27
-// note: we don't want to use atomics for signals because peer atomics are no
28
-// supported on PCIe links
27
+constexpr int kMaxBlocks = 36;
28
+// Counter may overflow, but it's fine since unsigned int overflow is
29
+// well-defined behavior.
30
+using FlagType = uint32_t;
31
struct Signal {
30
- alignas(128) uint32_t start[kMaxBlocks][8];
31
- alignas(128) uint32_t end[kMaxBlocks][8];
32
+ alignas(128) FlagType self_counter[kMaxBlocks][8];
33
+ // Two sets of peer counters are needed for two syncs. The reason is that
34
+ // it's possible for peer GPU block to arrive at the second sync point while
35
+ // the current GPU block haven't passed the first sync point. Thus, peer GPU
36
+ // may write counter+1 while current GPU is busy waiting for counter. We use
37
+ // alternating counter array to avoid this possibility.
38
+ alignas(128) FlagType peer_counter[2][kMaxBlocks][8];
39
};
40
41
struct __align__(16) RankData { const void* __restrict__ ptrs[8]; };
42
36
-struct __align__(16) RankSignals { volatile Signal* signals[8]; };
43
+struct __align__(16) RankSignals { Signal* signals[8]; };
44
45
// like std::array, but aligned
46
template <typename T, int sz>
@@ -123,47 +130,60 @@ DINLINE O downcast(array_t<float, O::size> val) {
130
}
131
}
132
126
-// This function is meant to be used as the first synchronization in the all
127
-// reduce kernel. Thus, it doesn't need to make any visibility guarantees for
128
-// prior memory accesses. Note: volatile writes will not be reordered against
129
-// other volatile writes.
130
-template <int ngpus>
131
-DINLINE void start_sync(const RankSignals& sg, volatile Signal* self_sg,
132
- int rank) {
133
- if (threadIdx.x < ngpus) {
134
- // reset flag for next time
135
- self_sg->end[blockIdx.x][threadIdx.x] = 0;
136
- // simultaneously write to the corresponding flag of all ranks.
137
- // Latency = 1 p2p write
138
- sg.signals[threadIdx.x]->start[blockIdx.x][rank] = 1;
139
- // wait until we got true from all ranks
140
- while (!self_sg->start[blockIdx.x][threadIdx.x]);
141
- }
142
- __syncthreads();
133
+static DINLINE void st_flag_release(FlagType* flag_addr, FlagType flag) {
134
+ asm volatile("st.release.sys.global.u32 [%1], %0;" ::"r"(flag),
135
+ "l"(flag_addr));
136
+}
137
+
138
+static DINLINE FlagType ld_flag_acquire(FlagType* flag_addr) {
139
+ FlagType flag;
140
+ asm volatile("ld.acquire.sys.global.u32 %0, [%1];"
141
+ : "=r"(flag)
142
+ : "l"(flag_addr));
143
+ return flag;
144
+}
145
+
146
+static DINLINE void st_flag_volatile(FlagType* flag_addr, FlagType flag) {
147
+ asm volatile("st.volatile.global.u32 [%1], %0;" ::"r"(flag), "l"(flag_addr));
148
+}
149
+
150
+static DINLINE FlagType ld_flag_volatile(FlagType* flag_addr) {
151
+ FlagType flag;
152
+ asm volatile("ld.volatile.global.u32 %0, [%1];"
153
+ : "=r"(flag)
154
+ : "l"(flag_addr));
155
+ return flag;
156
}
157
145
-// This function is meant to be used as the second or the final synchronization
146
-// barrier in the all reduce kernel. If it's the final synchronization barrier,
147
-// we don't need to make any visibility guarantees for prior memory accesses.
148
-template <int ngpus, bool final_sync = false>
149
-DINLINE void end_sync(const RankSignals& sg, volatile Signal* self_sg,
150
- int rank) {
151
- __syncthreads();
152
- // eliminate the case that prior writes are not visible after signals become
153
- // visible. Note that I did not managed to make this happen through a lot of
154
- // testing. Might be the case that hardware provides stronger guarantee than
155
- // the memory model.
156
- if constexpr (!final_sync) __threadfence_system();
158
+// is_start: whether this is the very first synchronization barrier.
159
+// need_fence: whether a memory fence is needed. If true, a release-acquire
160
+// semantic is used to enforce memory access order before and after this
161
+// barrier.
162
+template <int ngpus, bool is_start, bool need_fence = false>
163
+DINLINE void multi_gpu_barrier(const RankSignals& sg, Signal* self_sg,
164
+ int rank) {
165
+ if constexpr (!is_start) __syncthreads();
166
+ static_assert(
167
+ !(is_start && need_fence)); // Start barrier shouldn't need fence.
168
if (threadIdx.x < ngpus) {
158
- // reset flag for next time
159
- self_sg->start[blockIdx.x][threadIdx.x] = 0;
160
- // simultaneously write to the corresponding flag of all ranks.
161
- // Latency = 1 p2p write
162
- sg.signals[threadIdx.x]->end[blockIdx.x][rank] = 1;
163
- // wait until we got true from all ranks
164
- while (!self_sg->end[blockIdx.x][threadIdx.x]);
169
+ // Increment the counter. Technically we only need one counter, but we use
170
+ // multiple per block to eliminate the need to share the counter via smem.
171
+ auto val = self_sg->self_counter[blockIdx.x][threadIdx.x] += 1;
172
+ // Write the expected counter value to peer and wait for correct value from
173
+ // peer.
174
+ auto peer_counter_ptr =
175
+ &sg.signals[threadIdx.x]->peer_counter[val % 2][blockIdx.x][rank];
176
+ auto self_counter_ptr =
177
+ &self_sg->peer_counter[val % 2][blockIdx.x][threadIdx.x];
178
+ if constexpr (need_fence) {
179
+ st_flag_release(peer_counter_ptr, val);
180
+ while (ld_flag_acquire(self_counter_ptr) != val);
181
+ } else {
182
+ st_flag_volatile(peer_counter_ptr, val);
183
+ while (ld_flag_volatile(self_counter_ptr) != val);
184
+ }
185
}
166
- if constexpr (!final_sync) __syncthreads();
186
+ if constexpr (is_start || need_fence) __syncthreads();
187
}
188
189
template <typename P, int ngpus, typename A>
@@ -178,33 +198,31 @@ DINLINE P packed_reduce(const P* ptrs[], int idx) {
198
199
template <typename T, int ngpus>
200
__global__ void __launch_bounds__(512, 1)
181
- cross_device_reduce_1stage(RankData* _dp, RankSignals sg,
182
- volatile Signal* self_sg, T* __restrict__ result,
183
- int rank, int size) {
201
+ cross_device_reduce_1stage(RankData* _dp, RankSignals sg, Signal* self_sg,
202
+ T* __restrict__ result, int rank, int size) {
203
using P = typename packed_t<T>::P;
204
using A = typename packed_t<T>::A;
205
// note: we don't reorder the address so the accumulation order is the same
206
// for all ranks, ensuring bitwise identical results
207
auto dp = *_dp;
189
- start_sync<ngpus>(sg, self_sg, rank);
208
+ multi_gpu_barrier<ngpus, true>(sg, self_sg, rank);
209
// do the actual reduction
210
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
211
idx += gridDim.x * blockDim.x) {
212
((P*)result)[idx] = packed_reduce<P, ngpus, A>((const P**)&dp.ptrs[0], idx);
213
}
195
- end_sync<ngpus, true>(sg, self_sg, rank);
214
+ multi_gpu_barrier<ngpus, false>(sg, self_sg, rank);
215
}
216
217
template <typename P>
199
-DINLINE P* get_tmp_buf(volatile Signal* sg) {
218
+DINLINE P* get_tmp_buf(Signal* sg) {
219
return (P*)(((Signal*)sg) + 1);
220
}
221
222
template <typename T, int ngpus>
223
__global__ void __launch_bounds__(512, 1)
205
- cross_device_reduce_2stage(RankData* _dp, RankSignals sg,
206
- volatile Signal* self_sg, T* __restrict__ result,
207
- int rank, int size) {
224
+ cross_device_reduce_2stage(RankData* _dp, RankSignals sg, Signal* self_sg,
225
+ T* __restrict__ result, int rank, int size) {
226
int tid = blockIdx.x * blockDim.x + threadIdx.x;
227
int stride = gridDim.x * blockDim.x;
228
using P = typename packed_t<T>::P;
@@ -222,12 +240,12 @@ __global__ void __launch_bounds__(512, 1)
240
tmps[i] = get_tmp_buf<P>(sg.signals[target]);
241
}
242
auto tmp_out = tmps[0];
225
- start_sync<ngpus>(sg, self_sg, rank);
243
+ multi_gpu_barrier<ngpus, true>(sg, self_sg, rank);
244
// stage 1: reduce scatter
245
for (int idx = start + tid; idx < end; idx += stride) {
246
tmp_out[idx - start] = packed_reduce<P, ngpus, A>(ptrs, idx);
247
}
230
- end_sync<ngpus>(sg, self_sg, rank);
248
+ multi_gpu_barrier<ngpus, false, true>(sg, self_sg, rank);
249
250
// stage 2: allgather. Note: it's important to match the tid between
251
// the two stages, because visibility across devices is only guaranteed
@@ -437,6 +455,8 @@ class CustomAllreduce {
455
#define KL(ngpus, name) \
456
name<T, ngpus><<<blocks, threads, 0, stream>>>(ptrs, sg_, self_sg_, output, \
457
rank_, size);
458
+ // TODO(hanzhi713): Threshold is different for A100 and H100.
459
+ // Add per device threshold.
460
#define REDUCE_CASE(ngpus) \
461
case ngpus: { \
462
if (world_size_ == 2) { \
csrc/custom_all_reduce_test.cu
+9
-5
@@ -1,15 +1,15 @@
1
/**
2
* This is a standalone test for custom allreduce.
3
* To compile, make sure you have MPI and NCCL installed in your system.
4
- * export MPI_HOME=XXX
4
+ * export MPI_HOME=xxx
5
* nvcc -O2 -arch=native -std=c++17 custom_all_reduce_test.cu -o
6
- * custom_all_reduce_test -lnccl -I${MPI_HOME}/include -lmpi
6
+ * custom_all_reduce_test -lnccl -I${MPI_HOME} -lmpi
7
*
8
* Warning: this C++ test is not designed to be very readable and was used
9
* during the rapid prototyping process.
10
*
11
* To run:
12
- * mpirun -np 8 ./custom_all_reduce_test
12
+ * mpirun --allow-run-as-root -np 8 ./custom_all_reduce_test
13
*/
14
#include <cuda.h>
15
#include <curand_kernel.h>
@@ -302,15 +302,19 @@ int main(int argc, char** argv) {
302
303
bool performance_test = true;
304
cudaProfilerStart();
305
- // for (int threads : {256, 512}) {
305
+ // Uncomment to scan through different block size configs.
306
+ // for (int threads : {256, 512, 1024}) {
307
// for (int block_limit = 16; block_limit < 112; block_limit += 4) {
307
- // run<half>(myRank, nRanks, comm, threads, block_limit, 4096 * 1024);
308
+ // run<half>(myRank, nRanks, comm, threads, block_limit, 1024 * 1024,
309
+ // performance_test);
310
// }
311
// }
312
+ // Scan through different sizes to test performance.
313
for (int sz = 512; sz <= (8 << 20); sz *= 2) {
314
run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 47, performance_test);
315
}
316
317
cudaProfilerStop();
318
+ MPICHECK(MPI_Finalize());
319
return EXIT_SUCCESS;
320
}