@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 380170038e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Implement custom all reduce kernels (#2192)

Hanzhi Zhou committed Jan 28, 2024 at 04:46 UTC 380170038e05cf81953c29d7e8ed789e048b6434
18 files changed +1453 -65
csrc/custom_all_reduce.cu new
+148
@@ -0,0 +1,148 @@
1 +#include <ATen/cuda/Exceptions.h>
2 +#include <c10/cuda/CUDAGuard.h>
3 +#include <c10/cuda/CUDAStream.h>
4 +#include <torch/extension.h>
5 +
6 +#include "custom_all_reduce.cuh"
7 +
8 +// fake pointer type
9 +using fptr_t = uint64_t;
10 +static_assert(sizeof(void *) == sizeof(fptr_t));
11 +
12 +fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
13 + const std::vector<std::string> &handles,
14 + const std::vector<int64_t> &offsets, int rank,
15 + bool full_nvlink) {
16 + int world_size = offsets.size();
17 + if (world_size > 8)
18 + throw std::invalid_argument("world size > 8 is not supported");
19 + if (world_size % 2 != 0)
20 + throw std::invalid_argument("Odd num gpus is not supported for now");
21 + if (world_size != handles.size())
22 + throw std::invalid_argument(
23 + "handles length should equal to offsets length");
24 + if (rank < 0 || rank >= world_size)
25 + throw std::invalid_argument("invalid rank passed in");
26 +
27 + cudaIpcMemHandle_t ipc_handles[8];
28 + for (int i = 0; i < world_size; i++) {
29 + std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
30 + }
31 + return (fptr_t) new vllm::CustomAllreduce(
32 + reinterpret_cast<vllm::Metadata *>(meta.data_ptr()), rank_data.data_ptr(),
33 + rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
34 +}
35 +
36 +/**
37 + * Make sure tensor t's data lies completely within ((char)t.data_ptr()) +
38 + * t.numel() * t.element_size(). This is slightly weaker than t.is_contiguous()
39 + * because it allows transpose of contiguous slice (i.e. slicing the first
40 + * dimension). Currently, we require this because stride information is not
41 + * passed into the kernels and we treat input tensors as flat.
42 + *
43 + * Examples
44 + * A = torch.zeros(3, 3, 3)
45 + * 1. A: OK
46 + * 2. A[1:]: OK
47 + * 3. A.permute(2, 0, 1): OK
48 + * 4. A[1:].permute(2, 0, 1): OK
49 + * 5. A[None].expand(2, -1, -1, -1): Not OK
50 + * 6. A[:, 1:, 1:]: Not OK
51 + */
52 +bool _is_weak_contiguous(torch::Tensor &t) {
53 + return t.is_contiguous() ||
54 + (t.storage().nbytes() - t.storage_offset() * t.element_size() ==
55 + t.numel() * t.element_size());
56 +}
57 +
58 +bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
59 + bool full_nvlink) {
60 + auto inp_size = inp.numel() * inp.element_size();
61 + // custom allreduce requires input byte size to be multiples of 16
62 + if (inp_size % 16 != 0) return false;
63 + if (!_is_weak_contiguous(inp)) return false;
64 + if (world_size == 2 || full_nvlink) return inp_size <= max_size;
65 + // 4 PCIE GPUs use 2 stage allreduce, and is only faster than NCCL when size
66 + // <= 512k
67 + return world_size <= 4 && inp_size <= 512 * 1024;
68 +}
69 +
70 +void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
71 + cudaStream_t stream) {
72 + auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
73 + TORCH_CHECK(_is_weak_contiguous(out));
74 + switch (out.scalar_type()) {
75 + case at::ScalarType::Float: {
76 + fa->allreduce<float>(stream, reinterpret_cast<float *>(inp.data_ptr()),
77 + reinterpret_cast<float *>(out.data_ptr()),
78 + out.numel());
79 + break;
80 + }
81 + case at::ScalarType::Half: {
82 + fa->allreduce<half>(stream, reinterpret_cast<half *>(inp.data_ptr()),
83 + reinterpret_cast<half *>(out.data_ptr()),
84 + out.numel());
85 + break;
86 + }
87 +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
88 + case at::ScalarType::BFloat16: {
89 + fa->allreduce<nv_bfloat16>(
90 + stream, reinterpret_cast<nv_bfloat16 *>(inp.data_ptr()),
91 + reinterpret_cast<nv_bfloat16 *>(out.data_ptr()), out.numel());
92 + break;
93 + }
94 +#endif
95 + default:
96 + throw std::runtime_error(
97 + "custom allreduce only supports float32, float16 and bfloat16");
98 + }
99 +}
100 +
101 +void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out) {
102 + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
103 + auto stream = c10::cuda::getCurrentCUDAStream().stream();
104 + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
105 + TORCH_CHECK_EQ(inp.numel(), out.numel());
106 + _all_reduce(_fa, inp, out, stream);
107 +}
108 +
109 +void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &reg_buffer,
110 + torch::Tensor &out) {
111 + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
112 + auto stream = c10::cuda::getCurrentCUDAStream().stream();
113 +
114 + auto input_size = inp.numel() * inp.element_size();
115 + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
116 + TORCH_CHECK_EQ(inp.numel(), out.numel());
117 + TORCH_CHECK(input_size <= reg_buffer.numel() * reg_buffer.element_size(),
118 + "registered buffer is too small to contain the input");
119 + AT_CUDA_CHECK(cudaMemcpyAsync(reg_buffer.data_ptr(), inp.data_ptr(),
120 + input_size, cudaMemcpyDeviceToDevice, stream));
121 + _all_reduce(_fa, reg_buffer, out, stream);
122 +}
123 +
124 +void dispose(fptr_t _fa) {
125 + auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
126 + delete fa;
127 +}
128 +
129 +int meta_size() { return sizeof(vllm::Metadata); }
130 +
131 +void register_buffer(fptr_t _fa, torch::Tensor &t,
132 + const std::vector<std::string> &handles,
133 + const std::vector<int64_t> &offsets) {
134 + auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
135 + fa->register_buffer(handles, offsets, t.data_ptr());
136 +}
137 +
138 +std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
139 + fptr_t _fa) {
140 + auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
141 + return fa->get_graph_buffer_ipc_meta();
142 +}
143 +
144 +void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
145 + const std::vector<std::vector<int64_t>> &offsets) {
146 + auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
147 + fa->register_graph_buffers(handles, offsets);
148 +}
csrc/custom_all_reduce.cuh new
+555
@@ -0,0 +1,555 @@
1 +#pragma once
2 +
3 +#include <cuda.h>
4 +#include <cuda_bf16.h>
5 +#include <cuda_fp16.h>
6 +#include <cuda_runtime.h>
7 +
8 +#include <iostream>
9 +#include <limits>
10 +#include <unordered_map>
11 +#include <vector>
12 +
13 +#define CUDACHECK(cmd) \
14 + do { \
15 + cudaError_t e = cmd; \
16 + if (e != cudaSuccess) { \
17 + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \
18 + cudaGetErrorString(e)); \
19 + exit(EXIT_FAILURE); \
20 + } \
21 + } while (0)
22 +
23 +namespace vllm {
24 +
25 +struct Signal {
26 + alignas(64) union {
27 + uint64_t flag;
28 + unsigned char data[8];
29 + } start;
30 + alignas(64) union {
31 + uint64_t flag;
32 + unsigned char data[8];
33 + } end;
34 +};
35 +
36 +struct Metadata {
37 + alignas(128) Signal sg;
38 + alignas(128) int counter;
39 +};
40 +static_assert(offsetof(Metadata, counter) == 128);
41 +static_assert(sizeof(Metadata) == 256);
42 +
43 +struct __align__(16) RankData { const void *__restrict__ ptrs[8]; };
44 +
45 +struct RankSignals {
46 + volatile Signal *signals[8];
47 +};
48 +
49 +// like std::array, but aligned
50 +template <typename T, int sz>
51 +struct __align__(alignof(T) * sz) array_t {
52 + T data[sz];
53 + using type = T;
54 + static constexpr int size = sz;
55 +};
56 +
57 +// use packed type to maximize memory efficiency
58 +// goal: generate ld.128 and st.128 instructions
59 +template <typename T>
60 +struct packed_t {
61 + // the (P)acked type for load/store
62 + using P = array_t<T, 16 / sizeof(T)>;
63 + // the (A)ccumulator type for reduction
64 + using A = array_t<float, 16 / sizeof(T)>;
65 +};
66 +
67 +#define DINLINE __device__ __forceinline__
68 +
69 +// scalar cast functions
70 +DINLINE float upcast_s(half val) { return __half2float(val); }
71 +
72 +template <typename T>
73 +DINLINE T downcast_s(float val);
74 +template <>
75 +DINLINE half downcast_s(float val) {
76 + return __float2half(val);
77 +}
78 +
79 +// scalar add functions
80 +// for some reason when compiling with Pytorch, the + operator for half and
81 +// bfloat is disabled so we call the intrinsics directly
82 +DINLINE half &assign_add(half &a, half b) {
83 + a = __hadd(a, b);
84 + return a;
85 +}
86 +DINLINE float &assign_add(float &a, float b) { return a += b; }
87 +
88 +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
89 +DINLINE float upcast_s(nv_bfloat16 val) { return __bfloat162float(val); }
90 +template <>
91 +DINLINE nv_bfloat16 downcast_s(float val) {
92 + return __float2bfloat16(val);
93 +}
94 +DINLINE nv_bfloat16 &assign_add(nv_bfloat16 &a, nv_bfloat16 b) {
95 + a = __hadd(a, b);
96 + return a;
97 +}
98 +#endif
99 +
100 +template <typename T, int N>
101 +DINLINE array_t<T, N> &packed_assign_add(array_t<T, N> &a, array_t<T, N> b) {
102 +#pragma unroll
103 + for (int i = 0; i < N; i++) {
104 + assign_add(a.data[i], b.data[i]);
105 + }
106 + return a;
107 +}
108 +
109 +template <typename T, int N>
110 +DINLINE array_t<float, N> upcast(array_t<T, N> val) {
111 + if constexpr (std::is_same<T, float>::value) {
112 + return val;
113 + } else {
114 + array_t<float, N> out;
115 +#pragma unroll
116 + for (int i = 0; i < N; i++) {
117 + out.data[i] = upcast_s(val.data[i]);
118 + }
119 + return out;
120 + }
121 +}
122 +
123 +template <typename O>
124 +DINLINE O downcast(array_t<float, O::size> val) {
125 + if constexpr (std::is_same<typename O::type, float>::value) {
126 + return val;
127 + } else {
128 + O out;
129 +#pragma unroll
130 + for (int i = 0; i < O::size; i++) {
131 + out.data[i] = downcast_s<typename O::type>(val.data[i]);
132 + }
133 + return out;
134 + }
135 +}
136 +
137 +// compute flag at compile time
138 +__host__ __device__ constexpr uint64_t compute_flag(int ngpus) {
139 + auto m = std::numeric_limits<uint64_t>::max();
140 + return m >> ((8 - ngpus) * 8);
141 +}
142 +
143 +template <int ngpus>
144 +DINLINE void start_sync(const RankSignals &sg, volatile Metadata *meta,
145 + int rank) {
146 + constexpr auto FLAG = compute_flag(ngpus);
147 + if (blockIdx.x == 0) {
148 + if (threadIdx.x < ngpus)
149 + // simultaneously write to the corresponding byte to all other ranks.
150 + // Latency = 1 p2p write
151 + sg.signals[threadIdx.x]->start.data[rank] = 255;
152 + else if (threadIdx.x == 32)
153 + // reset
154 + meta->sg.end.flag = 0;
155 + }
156 + if (threadIdx.x == 0) {
157 + while (meta->sg.start.flag != FLAG)
158 + ;
159 + }
160 + __syncthreads();
161 +}
162 +
163 +template <int ngpus, bool final_sync = false>
164 +DINLINE void end_sync(const RankSignals &sg, volatile Metadata *meta,
165 + int rank) {
166 + constexpr auto FLAG = compute_flag(ngpus);
167 + __syncthreads();
168 + __shared__ int num;
169 + if (threadIdx.x == 0) num = atomicAdd((int *)&meta->counter, 1);
170 + __syncthreads();
171 +
172 + // Only the last completing block can perform the end synchronization
173 + // This can ensures when the final busy wait ends, all ranks must have
174 + // finished reading each other's buffer.
175 + if (num == gridDim.x - 1) {
176 + if (threadIdx.x == 32) {
177 + // reset in a different warp
178 + meta->counter = 0;
179 + meta->sg.start.flag = 0;
180 + } else if (threadIdx.x < ngpus) {
181 + // simultaneously write to the corresponding byte to all other ranks.
182 + // Latency = 1 p2p write
183 + sg.signals[threadIdx.x]->end.data[rank] = 255;
184 + }
185 + // if this is the final sync, only one block needs it
186 + // because kernel exit can serve as sync
187 + if constexpr (final_sync) {
188 + if (threadIdx.x == 0) {
189 + while (meta->sg.end.flag != FLAG)
190 + ;
191 + }
192 + }
193 + }
194 + if constexpr (!final_sync) {
195 + if (threadIdx.x == 0) {
196 + while (meta->sg.end.flag != FLAG)
197 + ;
198 + }
199 + __syncthreads();
200 + }
201 +}
202 +
203 +template <typename P, int ngpus, typename A>
204 +DINLINE P packed_reduce(const P *ptrs[], int idx) {
205 + A tmp = upcast(ptrs[0][idx]);
206 +#pragma unroll
207 + for (int i = 1; i < ngpus; i++) {
208 + packed_assign_add(tmp, upcast(ptrs[i][idx]));
209 + }
210 + return downcast<P>(tmp);
211 +}
212 +
213 +template <typename T, int ngpus>
214 +__global__ void __launch_bounds__(512, 1)
215 + cross_device_reduce_1stage(RankData *_dp, RankSignals sg,
216 + volatile Metadata *meta, T *__restrict__ result,
217 + int rank, int size) {
218 + using P = typename packed_t<T>::P;
219 + using A = typename packed_t<T>::A;
220 + // note: we don't reorder the address so the accumulation order is the same
221 + // for all ranks, ensuring bitwise identical results
222 + auto dp = *_dp;
223 + start_sync<ngpus>(sg, meta, rank);
224 + // do the actual reduction
225 + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
226 + idx += gridDim.x * blockDim.x) {
227 + ((P *)result)[idx] =
228 + packed_reduce<P, ngpus, A>((const P **)&dp.ptrs[0], idx);
229 + }
230 + end_sync<ngpus, true>(sg, meta, rank);
231 +}
232 +
233 +template <typename P>
234 +DINLINE P *get_tmp_buf(volatile Signal *sg) {
235 + return (P *)(((Metadata *)sg) + 1);
236 +}
237 +
238 +template <typename T, int ngpus>
239 +__global__ void __launch_bounds__(512, 1)
240 + cross_device_reduce_2stage(RankData *_dp, RankSignals sg,
241 + volatile Metadata *meta, T *__restrict__ result,
242 + int rank, int size) {
243 + int tid = blockIdx.x * blockDim.x + threadIdx.x;
244 + int stride = gridDim.x * blockDim.x;
245 + using P = typename packed_t<T>::P;
246 + using A = typename packed_t<T>::A;
247 + int part = size / ngpus;
248 + int start = rank * part;
249 + int end = rank == ngpus - 1 ? size : start + part;
250 + const P *ptrs[ngpus];
251 + P *tmps[ngpus];
252 +#pragma unroll
253 + for (int i = 0; i < ngpus; i++) {
254 + int target = (rank + i) % ngpus;
255 + ptrs[i] = (const P *)_dp->ptrs[target];
256 + tmps[i] = get_tmp_buf<P>(sg.signals[target]);
257 + }
258 + auto tmp_out = tmps[0];
259 + start_sync<ngpus>(sg, meta, rank);
260 + // stage 1: reduce scatter
261 + for (int idx = start + tid; idx < end; idx += stride) {
262 + tmp_out[idx - start] = packed_reduce<P, ngpus, A>(ptrs, idx);
263 + }
264 + // Maybe TODO: replace this with per-block release-acquire
265 + // can save about 1-2us (not a lot though)
266 + end_sync<ngpus>(sg, meta, rank);
267 +
268 + // stage 2: allgather
269 + for (int idx = tid; idx < part; idx += stride) {
270 +#pragma unroll
271 + for (int i = 0; i < ngpus; i++) {
272 + int dst_idx = ((rank + i) % ngpus) * part + idx;
273 + ((P *)result)[dst_idx] = tmps[i][idx];
274 + }
275 + }
276 + // process the last larger partition
277 + int remaining = size - part * ngpus;
278 + if (tid < remaining) {
279 + int dst_idx = tid + part * ngpus;
280 + ((P *)result)[dst_idx] = get_tmp_buf<P>(sg.signals[ngpus - 1])[part + tid];
281 + }
282 +
283 + // faster than this
284 + // for (int idx = tid; idx < size; idx += stride) {
285 + // int target_rank = idx / part;
286 + // if (target_rank == ngpus) target_rank -= 1;
287 + // ((P *)result)[idx] = tmps[target_rank][idx - target_rank * part];
288 + // }
289 +}
290 +
291 +template <typename T, int ngpus>
292 +__global__ void __launch_bounds__(512, 1)
293 + cross_device_reduce_half_butterfly(RankData *_dp, RankSignals sg,
294 + volatile Metadata *meta,
295 + T *__restrict__ result, int rank,
296 + int size) {
297 + int tid = blockIdx.x * blockDim.x + threadIdx.x;
298 + int stride = gridDim.x * blockDim.x;
299 + using P = typename packed_t<T>::P;
300 + using A = typename packed_t<T>::A;
301 + auto tmp_out = get_tmp_buf<P>(sg.signals[rank]);
302 + constexpr int hg = ngpus / 2;
303 + // Actually not quite half butterfly.
304 + // This is an all-to-all within each group containing half of the ranks
305 + // followed by cross-group add. Equivalent to half butterfly when there
306 + // are 4 GPUs, a common case for PCIe cards like T4 and A10.
307 + const P *ptrs[hg];
308 + {
309 + int start = rank - rank % hg;
310 +#pragma unroll
311 + for (int i = 0; i < hg; i++) {
312 + ptrs[i] = (const P *)_dp->ptrs[i + start];
313 + }
314 + }
315 + start_sync<ngpus>(sg, meta, rank);
316 + for (int idx = tid; idx < size; idx += stride) {
317 + tmp_out[idx] = packed_reduce<P, hg, A>(ptrs, idx);
318 + }
319 + end_sync<ngpus>(sg, meta, rank);
320 +
321 + auto src = get_tmp_buf<P>(sg.signals[(ngpus - 1) - rank % ngpus]);
322 + // do the cross group reduction
323 + for (int idx = tid; idx < size; idx += stride) {
324 + auto tmp = tmp_out[idx];
325 + packed_assign_add(tmp, src[idx]);
326 + ((P *)result)[idx] = tmp;
327 + }
328 +}
329 +
330 +class CustomAllreduce {
331 + public:
332 + int rank_;
333 + int world_size_;
334 + bool full_nvlink_;
335 +
336 + // below are device pointers
337 + RankSignals sg_;
338 + std::unordered_map<void *, RankData *> buffers_;
339 + Metadata *meta_;
340 +
341 + // stores the registered device pointers from all ranks
342 + RankData *d_rank_data_base_, *d_rank_data_end_;
343 + std::vector<void *> graph_unreg_buffers_;
344 + std::vector<void *> ipc_handles_;
345 +
346 + /**
347 + * meta is a pointer to device metadata and temporary buffer for allreduce.
348 + *
349 + * There's a total of sizeof(Metadata) of prefix before the actual data,
350 + * so meta + 1 points to actual temporary buffer.
351 + *
352 + * note: this class does not own any device memory. Any required buffers
353 + * are passed in from the constructor
354 + */
355 + CustomAllreduce(Metadata *meta, void *rank_data, size_t rank_data_sz,
356 + const cudaIpcMemHandle_t *handles,
357 + const std::vector<int64_t> &offsets, int rank,
358 + bool full_nvlink = true)
359 + : rank_(rank),
360 + world_size_(offsets.size()),
361 + full_nvlink_(full_nvlink),
362 + meta_(meta),
363 + d_rank_data_base_(reinterpret_cast<RankData *>(rank_data)),
364 + d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
365 + for (int i = 0; i < world_size_; i++) {
366 + Metadata *rank_meta;
367 + if (i != rank_) {
368 + char *handle;
369 + CUDACHECK(cudaIpcOpenMemHandle((void **)&handle, handles[i],
370 + cudaIpcMemLazyEnablePeerAccess));
371 + ipc_handles_.push_back(handle);
372 + handle += offsets[i];
373 + rank_meta = (Metadata *)handle;
374 + } else {
375 + rank_meta = meta_;
376 + }
377 + sg_.signals[i] = &rank_meta->sg;
378 + }
379 + }
380 +
381 + std::pair<std::vector<uint8_t>, std::vector<int64_t>>
382 + get_graph_buffer_ipc_meta() {
383 + auto num_buffers = graph_unreg_buffers_.size();
384 + auto handle_sz = sizeof(cudaIpcMemHandle_t);
385 + std::vector<uint8_t> handles(handle_sz * num_buffers, 0);
386 + std::vector<int64_t> offsets(num_buffers);
387 + for (int i = 0; i < num_buffers; i++) {
388 + auto ptr = graph_unreg_buffers_[i];
389 + void *base_ptr;
390 + // note: must share the base address of each allocation, or we get wrong
391 + // address
392 + if (cuPointerGetAttribute(&base_ptr,
393 + CU_POINTER_ATTRIBUTE_RANGE_START_ADDR,
394 + (CUdeviceptr)ptr) != CUDA_SUCCESS)
395 + throw std::runtime_error("failed to get pointer attr");
396 + CUDACHECK(cudaIpcGetMemHandle(
397 + (cudaIpcMemHandle_t *)&handles[i * handle_sz], base_ptr));
398 + offsets[i] = ((char *)ptr) - ((char *)base_ptr);
399 + }
400 + return std::make_pair(handles, offsets);
401 + }
402 +
403 + void check_rank_data_capacity(size_t num = 1) {
404 + if (d_rank_data_base_ + num > d_rank_data_end_)
405 + throw std::runtime_error(
406 + "Rank data buffer is overflowed by " +
407 + std::to_string(d_rank_data_base_ + num - d_rank_data_end_));
408 + }
409 +
410 + void register_buffer(const std::vector<std::string> &handles,
411 + const std::vector<int64_t> &offsets, void *self) {
412 + check_rank_data_capacity();
413 + RankData data;
414 + for (int i = 0; i < world_size_; i++) {
415 + if (i != rank_) {
416 + char *handle;
417 + CUDACHECK(cudaIpcOpenMemHandle(
418 + (void **)&handle, *((const cudaIpcMemHandle_t *)handles[i].data()),
419 + cudaIpcMemLazyEnablePeerAccess));
420 + ipc_handles_.push_back(handle);
421 + handle += offsets[i];
422 + data.ptrs[i] = handle;
423 + } else {
424 + data.ptrs[i] = self;
425 + }
426 + }
427 + auto d_data = d_rank_data_base_++;
428 + CUDACHECK(
429 + cudaMemcpy(d_data, &data, sizeof(RankData), cudaMemcpyHostToDevice));
430 + buffers_[self] = d_data;
431 + }
432 +
433 + // note: when registering graph buffers, we intentionally choose to not
434 + // deduplicate the addresses. That means if the allocator reuses some
435 + // addresses, they will be registered again. This is to account for the remote
436 + // possibility of different allocation patterns between ranks. For example,
437 + // rank 1 may get the same input address for the second allreduce, but rank 2
438 + // got a different address. IPC handles have internal reference counting
439 + // mechanism so overhead should be small.
440 + void register_graph_buffers(
441 + const std::vector<std::string> &handles,
442 + const std::vector<std::vector<int64_t>> &offsets) {
443 + auto num_buffers = graph_unreg_buffers_.size();
444 + check_rank_data_capacity(num_buffers);
445 + std::vector<RankData> rank_data(num_buffers);
446 + for (int i = 0; i < num_buffers; i++) {
447 + auto self_ptr = graph_unreg_buffers_[i];
448 + auto &rd = rank_data[i];
449 + for (int j = 0; j < world_size_; j++) {
450 + if (j != rank_) {
451 + char *handle;
452 + CUDACHECK(cudaIpcOpenMemHandle(
453 + (void **)&handle,
454 + *((cudaIpcMemHandle_t *)&handles[j]
455 + [i * sizeof(cudaIpcMemHandle_t)]),
456 + cudaIpcMemLazyEnablePeerAccess));
457 + ipc_handles_.push_back(handle);
458 + handle += offsets[j][i];
459 + rd.ptrs[j] = handle;
460 + } else {
461 + rd.ptrs[j] = self_ptr;
462 + }
463 + }
464 + }
465 + CUDACHECK(cudaMemcpy(d_rank_data_base_, rank_data.data(),
466 + sizeof(RankData) * num_buffers,
467 + cudaMemcpyHostToDevice));
468 + d_rank_data_base_ += num_buffers;
469 + graph_unreg_buffers_.clear();
470 + }
471 +
472 + /**
473 + * This is the result after careful grid search. Using 36 blocks give the best
474 + * or close to the best runtime on the devices I tried: A100, A10, A30, T4,
475 + * V100. You'll notice that NCCL kernels also only take a small amount of SMs.
476 + * Not quite sure the underlying reason, but my guess is that too many SMs
477 + * will cause contention on NVLink bus.
478 + */
479 + template <typename T>
480 + void allreduce(cudaStream_t stream, T *input, T *output, int size,
481 + int threads = 512, int block_limit = 36) {
482 + auto d = packed_t<T>::P::size;
483 + if (size % d != 0)
484 + throw std::runtime_error(
485 + "custom allreduce currently requires input length to be multiple "
486 + "of " +
487 + std::to_string(d));
488 +
489 + RankData *ptrs;
490 + cudaStreamCaptureStatus status;
491 + CUDACHECK(cudaStreamIsCapturing(stream, &status));
492 + if (status == cudaStreamCaptureStatusActive) {
493 + ptrs = d_rank_data_base_ + graph_unreg_buffers_.size();
494 + graph_unreg_buffers_.push_back(input);
495 + } else {
496 + auto it = buffers_.find(input);
497 + if (it == buffers_.end())
498 + throw std::runtime_error(
499 + "buffer address " +
500 + std::to_string(reinterpret_cast<uint64_t>(input)) +
501 + " is not registered!");
502 + ptrs = it->second;
503 + }
504 +
505 + size /= d;
506 + auto bytes = size * sizeof(typename packed_t<T>::P);
507 + int blocks = std::min(block_limit, (size + threads - 1) / threads);
508 +#define KL(ngpus, name) \
509 + name<T, ngpus> \
510 + <<<blocks, threads, 0, stream>>>(ptrs, sg_, meta_, output, rank_, size);
511 +#define REDUCE_CASE(ngpus) \
512 + case ngpus: { \
513 + if (world_size_ == 2) { \
514 + KL(ngpus, cross_device_reduce_1stage); \
515 + } else if (full_nvlink_) { \
516 + if ((world_size_ <= 4 && bytes < 512 * 1024) || \
517 + (world_size_ <= 8 && bytes < 256 * 1024)) { \
518 + KL(ngpus, cross_device_reduce_1stage); \
519 + } else { \
520 + KL(ngpus, cross_device_reduce_2stage); \
521 + } \
522 + } else { \
523 + KL(ngpus, cross_device_reduce_half_butterfly); \
524 + } \
525 + break; \
526 + }
527 +
528 + switch (world_size_) {
529 + REDUCE_CASE(2)
530 + REDUCE_CASE(4)
531 + REDUCE_CASE(6)
532 + REDUCE_CASE(8)
533 + default:
534 + throw std::runtime_error(
535 + "custom allreduce only supports num gpus in (2,4,6,8). Actual num "
536 + "gpus = " +
537 + std::to_string(world_size_));
538 + }
539 +#undef REDUCE_CASE
540 +#undef KL
541 + }
542 +
543 + ~CustomAllreduce() {
544 + for (auto ptr : ipc_handles_) {
545 + CUDACHECK(cudaIpcCloseMemHandle(ptr));
546 + }
547 + }
548 +};
549 +/**
550 + * To inspect PTX/SASS, copy paste this header file to compiler explorer and add
551 + a template instantiation:
552 + * template void CustomAllreduce::allreduce<half>(cudaStream_t, half *, half *,
553 + int, int, int);
554 +*/
555 +} // namespace vllm
csrc/custom_all_reduce_test.cu new
+284
@@ -0,0 +1,284 @@
1 +/**
2 + * This is a standalone test for custom allreduce.
3 + * To compile, make sure you have MPI and NCCL installed in your system.
4 + * export MPI_HOME=XXX
5 + * nvcc -O2 -arch=native -std=c++17 custom_all_reduce_test.cu -o
6 + * custom_all_reduce_test -lnccl -I${MPI_HOME}/include -lmpi
7 + *
8 + * Warning: this C++ test is not designed to be very readable and was used
9 + * during the rapid prototyping process.
10 + *
11 + * To run:
12 + * mpirun -np 8 ./custom_all_reduce_test
13 + */
14 +#include <cuda.h>
15 +#include <curand_kernel.h>
16 +#include <stdio.h>
17 +#include <stdlib.h>
18 +
19 +#include <limits>
20 +#include <vector>
21 +
22 +#include "cuda_profiler_api.h"
23 +#include "custom_all_reduce.cuh"
24 +#include "mpi.h"
25 +#include "nccl.h"
26 +
27 +#define MPICHECK(cmd) \
28 + do { \
29 + int e = cmd; \
30 + if (e != MPI_SUCCESS) { \
31 + printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \
32 + exit(EXIT_FAILURE); \
33 + } \
34 + } while (0)
35 +
36 +#define NCCLCHECK(cmd) \
37 + do { \
38 + ncclResult_t r = cmd; \
39 + if (r != ncclSuccess) { \
40 + printf("Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \
41 + ncclGetErrorString(r)); \
42 + exit(EXIT_FAILURE); \
43 + } \
44 + } while (0)
45 +
46 +__global__ void dummy_kernel() {
47 + for (int i = 0; i < 100; i++) __nanosleep(1000000); // 100ms
48 +}
49 +
50 +template <typename T>
51 +__global__ void set_data(T *data, int size, int myRank) {
52 + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
53 + idx += gridDim.x * blockDim.x) {
54 + data[idx] = myRank * 0.11f;
55 + }
56 +}
57 +
58 +template <typename T>
59 +__global__ void convert_data(const T *data1, const T *data2, double *fdata1,
60 + double *fdata2, int size) {
61 + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
62 + idx += gridDim.x * blockDim.x) {
63 + fdata1[idx] = data1[idx];
64 + fdata2[idx] = data2[idx];
65 + }
66 +}
67 +
68 +__global__ void init_rand(curandState_t *state, int size, int nRanks) {
69 + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
70 + idx += gridDim.x * blockDim.x) {
71 + for (int i = 0; i < nRanks; i++) {
72 + curand_init(i + 1, idx, 0, &state[idx * nRanks + i]);
73 + }
74 + }
75 +}
76 +
77 +template <typename T>
78 +__global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
79 + int myRank, int nRanks, int size) {
80 + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
81 + idx += gridDim.x * blockDim.x) {
82 + double sum = 0.0;
83 + for (int i = 0; i < nRanks; i++) {
84 + double val = curand_uniform_double(&state[idx * nRanks + i]) * 4;
85 + T hval = val; // downcast first
86 + sum += static_cast<double>(hval);
87 + if (i == myRank) data[idx] = hval;
88 + }
89 + ground_truth[idx] = sum;
90 + }
91 +}
92 +
93 +template <typename T>
94 +void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
95 + int data_size) {
96 + T *result;
97 + cudaStream_t stream;
98 + CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
99 + CUDACHECK(cudaMalloc(&result, data_size * sizeof(T)));
100 + CUDACHECK(cudaMemset(result, 0, data_size * sizeof(T)));
101 +
102 + cudaIpcMemHandle_t self_data_handle;
103 + cudaIpcMemHandle_t data_handles[8];
104 + vllm::Metadata *buffer;
105 + T *self_data_copy;
106 + /**
107 + * Allocate IPC buffer
108 + *
109 + * The first section is a temporary buffer for storing intermediate allreduce
110 + * results, if a particular algorithm requires it. The second section is for
111 + * the input to the allreduce. The actual API takes the input pointer as an
112 + * argument (that is, they can and usually should be allocated separately).
113 + * But since the input pointers and the temporary buffer all require IPC
114 + * registration, they are allocated and registered together in the test for
115 + * convenience.
116 + */
117 + CUDACHECK(
118 + cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
119 + CUDACHECK(cudaMemset(buffer, 0,
120 + 2 * data_size * sizeof(T) + sizeof(vllm::Metadata)));
121 + CUDACHECK(cudaMalloc(&self_data_copy, data_size * sizeof(T)));
122 + CUDACHECK(cudaIpcGetMemHandle(&self_data_handle, buffer));
123 +
124 + MPICHECK(MPI_Allgather(&self_data_handle, sizeof(cudaIpcMemHandle_t),
125 + MPI_BYTE, data_handles, sizeof(cudaIpcMemHandle_t),
126 + MPI_BYTE, MPI_COMM_WORLD));
127 +
128 + void *rank_data;
129 + size_t rank_data_sz = 16 * 1024 * 1024;
130 + CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
131 + std::vector<int64_t> offsets(nRanks, 0);
132 + vllm::CustomAllreduce fa(buffer, rank_data, rank_data_sz, data_handles,
133 + offsets, myRank);
134 + auto *self_data =
135 + reinterpret_cast<T *>(reinterpret_cast<char *>(buffer) +
136 + sizeof(vllm::Metadata) + data_size * sizeof(T));
137 + // hack buffer registration
138 + {
139 + std::vector<std::string> handles;
140 + handles.reserve(nRanks);
141 + for (int i = 0; i < nRanks; i++) {
142 + char *begin = (char *)&data_handles[i];
143 + char *end = (char *)&data_handles[i + 1];
144 + handles.emplace_back(begin, end);
145 + }
146 + std::vector<int64_t> offsets(
147 + nRanks, sizeof(vllm::Metadata) + data_size * sizeof(T));
148 + fa.register_buffer(handles, offsets, self_data);
149 + }
150 +
151 + double *ground_truth;
152 + CUDACHECK(cudaMallocHost(&ground_truth, data_size * sizeof(double)));
153 + curandState_t *states;
154 + CUDACHECK(cudaMalloc(&states, sizeof(curandState_t) * nRanks * data_size));
155 + init_rand<<<108, 1024, 0, stream>>>(states, data_size, nRanks);
156 + gen_data<T><<<108, 1024, 0, stream>>>(states, self_data, ground_truth, myRank,
157 + nRanks, data_size);
158 + CUDACHECK(cudaMemcpyAsync(self_data_copy, self_data, data_size * sizeof(T),
159 + cudaMemcpyDeviceToDevice, stream));
160 + cudaEvent_t start, stop;
161 + CUDACHECK(cudaEventCreate(&start));
162 + CUDACHECK(cudaEventCreate(&stop));
163 +
164 + ncclDataType_t ncclDtype;
165 + if (std::is_same<T, half>::value) {
166 + ncclDtype = ncclFloat16;
167 + } else if (std::is_same<T, nv_bfloat16>::value) {
168 + ncclDtype = ncclBfloat16;
169 + } else {
170 + ncclDtype = ncclFloat;
171 + }
172 +
173 + dummy_kernel<<<1, 1, 0, stream>>>();
174 + constexpr int warmup_iters = 5;
175 + constexpr int num_iters = 25;
176 + // warmup
177 + for (int i = 0; i < warmup_iters; i++) {
178 + NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
179 + stream));
180 + }
181 + CUDACHECK(cudaEventRecord(start, stream));
182 + for (int i = 0; i < num_iters; i++) {
183 + NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum, comm,
184 + stream));
185 + }
186 + CUDACHECK(cudaEventRecord(stop, stream));
187 + CUDACHECK(cudaStreamSynchronize(stream));
188 + float allreduce_ms = 0;
189 + cudaEventElapsedTime(&allreduce_ms, start, stop);
190 +
191 + // if (myRank == 1) dummy_kernel<<<1, 1, 0, stream>>>();
192 + // set_data<T><<<16, 1024, 0, stream>>>(self_data, data_size, myRank);
193 +
194 + dummy_kernel<<<1, 1, 0, stream>>>();
195 + // warm up
196 + for (int i = 0; i < warmup_iters; i++) {
197 + fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
198 + }
199 + CUDACHECK(cudaEventRecord(start, stream));
200 + for (int i = 0; i < num_iters; i++) {
201 + fa.allreduce<T>(stream, self_data, result, data_size, threads, block_limit);
202 + }
203 + CUDACHECK(cudaEventRecord(stop, stream));
204 + CUDACHECK(cudaStreamSynchronize(stream));
205 +
206 + float duration_ms = 0;
207 + cudaEventElapsedTime(&duration_ms, start, stop);
208 + if (myRank == 0)
209 + printf(
210 + "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
211 + "time:%.2fus\n",
212 + myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
213 + duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
214 +
215 + // And wait for all the queued up work to complete
216 + CUDACHECK(cudaStreamSynchronize(stream));
217 +
218 + NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
219 + ncclSum, comm, stream));
220 +
221 + double *nccl_result, *my_result;
222 + CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
223 + CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
224 +
225 + convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
226 + my_result, data_size);
227 + CUDACHECK(cudaStreamSynchronize(stream));
228 +
229 + for (unsigned long j = 0; j < data_size; j++) {
230 + auto diff = abs(nccl_result[j] - my_result[j]);
231 + if (diff >= 1e-2) {
232 + printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
233 + myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
234 + break;
235 + }
236 + }
237 +
238 + long double nccl_diffs = 0.0;
239 + long double my_diffs = 0.0;
240 + for (int j = 0; j < data_size; j++) {
241 + nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
242 + my_diffs += abs(my_result[j] - ground_truth[j]);
243 + }
244 + if (myRank == 0)
245 + std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
246 + << " me: " << my_diffs / data_size << std::endl;
247 +
248 + CUDACHECK(cudaFree(result));
249 + CUDACHECK(cudaFree(self_data_copy));
250 + CUDACHECK(cudaFree(rank_data));
251 + CUDACHECK(cudaFree(buffer));
252 + CUDACHECK(cudaFree(states));
253 + CUDACHECK(cudaFreeHost(ground_truth));
254 + CUDACHECK(cudaFreeHost(nccl_result));
255 + CUDACHECK(cudaFreeHost(my_result));
256 + CUDACHECK(cudaStreamDestroy(stream));
257 +}
258 +
259 +int main(int argc, char **argv) {
260 + int nRanks, myRank;
261 + MPICHECK(MPI_Init(&argc, &argv));
262 + MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
263 + MPICHECK(MPI_Comm_size(MPI_COMM_WORLD, &nRanks));
264 + CUDACHECK(cudaSetDevice(myRank));
265 + ncclUniqueId id;
266 + ncclComm_t comm;
267 + if (myRank == 0) ncclGetUniqueId(&id);
268 + MPICHECK(MPI_Bcast(static_cast<void *>(&id), sizeof(id), MPI_BYTE, 0,
269 + MPI_COMM_WORLD));
270 + NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
271 +
272 + cudaProfilerStart();
273 + // for (int threads : {256, 512}) {
274 + // for (int block_limit = 16; block_limit < 112; block_limit += 4) {
275 + // run<half>(myRank, nRanks, comm, threads, block_limit, 4096 * 1024);
276 + // }
277 + // }
278 + for (int sz = 512; sz <= (32 << 20); sz *= 2) {
279 + run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 50);
280 + }
281 +
282 + cudaProfilerStop();
283 + return EXIT_SUCCESS;
284 +}
csrc/ops.h
+22
@@ -97,3 +97,25 @@ torch::Tensor gptq_gemm(
97 void gptq_shuffle(
98 torch::Tensor q_weight,
99 torch::Tensor q_perm);
100 +
101 +
102 +#ifndef USE_ROCM
103 +using fptr_t = uint64_t;
104 +fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
105 + const std::vector<std::string> &handles,
106 + const std::vector<int64_t> &offsets, int rank,
107 + bool full_nvlink);
108 +bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
109 + bool full_nvlink);
110 +void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out);
111 +void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &reg_buffer,
112 + torch::Tensor &out);
113 +void dispose(fptr_t _fa);
114 +int meta_size();
115 +void register_buffer(fptr_t _fa, torch::Tensor &t,
116 + const std::vector<std::string> &handles,
117 + const std::vector<int64_t> &offsets);
118 +std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(fptr_t _fa);
119 +void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
120 + const std::vector<std::vector<int64_t>> &offsets);
121 +#endif
csrc/pybind.cpp
+16
@@ -88,4 +88,20 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
88 &get_max_shared_memory_per_block_device_attribute,
89 "Gets the maximum shared memory per block device attribute.");
90
91 +#ifndef USE_ROCM
92 + // Custom all-reduce kernels
93 + pybind11::module custom_ar = m.def_submodule("custom_ar", "custom allreduce");
94 + custom_ar.def("init_custom_ar", &init_custom_ar, "init_custom_ar");
95 + custom_ar.def("should_custom_ar", &should_custom_ar, "should_custom_ar");
96 + custom_ar.def("all_reduce_reg", &all_reduce_reg, "all_reduce_reg");
97 + custom_ar.def("all_reduce_unreg", &all_reduce_unreg, "all_reduce_unreg");
98 + custom_ar.def("dispose", &dispose, "dispose");
99 + custom_ar.def("meta_size", &meta_size, "meta_size");
100 + custom_ar.def("register_buffer", &register_buffer, "register_buffer");
101 + custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta,
102 + "get_graph_buffer_ipc_meta");
103 + custom_ar.def("register_graph_buffers", &register_graph_buffers,
104 + "register_graph_buffers");
105 +#endif
106 +
107 }
requirements.txt
+1
@@ -10,3 +10,4 @@ fastapi
10 uvicorn[standard]
11 pydantic >= 2.0 # Required for OpenAI server.
12 aioprometheus[starlette]
13 +pynvml == 11.5.0
setup.py
+4 -2
@@ -51,8 +51,8 @@ if _is_hip():
51 "Cannot find ROCM_HOME. ROCm must be available to build the package."
52 )
53 NVCC_FLAGS += ["-DUSE_ROCM"]
54 - NVCC_FLAGS += [f"-U__HIP_NO_HALF_CONVERSIONS__"]
55 - NVCC_FLAGS += [f"-U__HIP_NO_HALF_OPERATORS__"]
54 + NVCC_FLAGS += ["-U__HIP_NO_HALF_CONVERSIONS__"]
55 + NVCC_FLAGS += ["-U__HIP_NO_HALF_OPERATORS__"]
56
57 if _is_cuda() and CUDA_HOME is None:
58 raise RuntimeError(
@@ -307,6 +307,7 @@ vllm_extension_sources = [
307
308 if _is_cuda():
309 vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
310 + vllm_extension_sources.append("csrc/custom_all_reduce.cu")
311
312 if not _is_neuron():
313 vllm_extension = CUDAExtension(
@@ -316,6 +317,7 @@ if not _is_neuron():
317 "cxx": CXX_FLAGS,
318 "nvcc": NVCC_FLAGS,
319 },
320 + libraries=["cuda"] if _is_cuda() else [],
321 )
322 ext_modules.append(vllm_extension)
323
tests/distributed/test_comm_ops.py
+3 -27
@@ -6,25 +6,13 @@ import pytest
6 import torch
7 import ray
8
9 -from vllm.config import ParallelConfig
10 -from vllm.utils import get_open_port
9 from vllm.model_executor.parallel_utils.communication_op import (
10 tensor_model_parallel_all_reduce,
11 tensor_model_parallel_all_gather,
12 broadcast_tensor_dict,
13 )
16 -from vllm.worker.worker import _init_distributed_environment
17 -
18 -
19 -def init_test_distributed_environment(pipeline_parallel_size: int,
20 - tensor_parallel_size: int, rank: int,
21 - distributed_init_port: str):
22 - parallel_config = ParallelConfig(pipeline_parallel_size,
23 - tensor_parallel_size,
24 - worker_use_ray=True)
25 - distributed_init_method = f"tcp://localhost:{distributed_init_port}"
26 - _init_distributed_environment(parallel_config, rank,
27 - distributed_init_method)
14 +from vllm.test_utils import (init_test_distributed_environment,
15 + multi_process_tensor_parallel)
16
17
18 @ray.remote(num_gpus=1, max_calls=1)
@@ -101,16 +89,4 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
89 broadcast_tensor_dict_test_worker
90 ])
91 def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
104 - # Using ray helps debugging the error when it failed
105 - # as compared to multiprocessing.
106 - ray.init()
107 -
108 - distributed_init_port = get_open_port()
109 - refs = []
110 - for rank in range(tensor_parallel_size):
111 - refs.append(
112 - test_target.remote(tensor_parallel_size, rank,
113 - distributed_init_port))
114 - ray.get(refs)
115 -
116 - ray.shutdown()
92 + multi_process_tensor_parallel(tensor_parallel_size, test_target)
tests/distributed/test_custom_all_reduce.py new
+85
@@ -0,0 +1,85 @@
1 +import random
2 +
3 +import os
4 +import pytest
5 +import ray
6 +import torch
7 +import torch.distributed as dist
8 +
9 +from vllm.model_executor.parallel_utils import custom_all_reduce as custom_ar
10 +from vllm.model_executor.parallel_utils.communication_op import (
11 + tensor_model_parallel_all_reduce)
12 +from vllm.test_utils import (init_test_distributed_environment,
13 + multi_process_tensor_parallel)
14 +
15 +random.seed(42)
16 +test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
17 +for i, v in enumerate(test_sizes):
18 + test_sizes[i] -= v % 8
19 +
20 +
21 +@ray.remote(num_gpus=1, max_calls=1)
22 +def graph_allreduce(world_size, rank, distributed_init_port):
23 + del os.environ["CUDA_VISIBLE_DEVICES"]
24 + device = torch.device(f"cuda:{rank}")
25 + torch.cuda.set_device(device)
26 + init_test_distributed_environment(1, world_size, rank,
27 + distributed_init_port)
28 +
29 + custom_ar.init_custom_ar()
30 + for sz in test_sizes:
31 + for dtype in [torch.float32, torch.float16, torch.bfloat16]:
32 + with custom_ar.capture():
33 + # use integers so result matches NCCL exactly
34 + inp1 = torch.randint(1,
35 + 16, (sz, ),
36 + dtype=dtype,
37 + device=torch.cuda.current_device())
38 + inp2 = torch.randint(1,
39 + 16, (sz, ),
40 + dtype=dtype,
41 + device=torch.cuda.current_device())
42 + torch.cuda.synchronize()
43 + graph = torch.cuda.CUDAGraph()
44 + with torch.cuda.graph(graph):
45 + out1 = tensor_model_parallel_all_reduce(inp1)
46 + # the input buffer is immediately modified to test
47 + # synchronization
48 + dist.all_reduce(inp1)
49 + out2 = tensor_model_parallel_all_reduce(inp2)
50 + dist.all_reduce(inp2)
51 + graph.replay()
52 + assert torch.allclose(out1, inp1)
53 + assert torch.allclose(out2, inp2)
54 +
55 +
56 +@ray.remote(num_gpus=1, max_calls=1)
57 +def eager_allreduce(world_size, rank, distributed_init_port):
58 + del os.environ["CUDA_VISIBLE_DEVICES"]
59 + device = torch.device(f"cuda:{rank}")
60 + torch.cuda.set_device(device)
61 + init_test_distributed_environment(1, world_size, rank,
62 + distributed_init_port)
63 +
64 + sz = 1024
65 + custom_ar.init_custom_ar()
66 + fa = custom_ar.get_handle()
67 + inp = torch.ones(sz, dtype=torch.float32, device=device)
68 + out = fa.all_reduce_unreg(inp)
69 + assert torch.allclose(out, inp * world_size)
70 +
71 + inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
72 + out = fa.all_reduce_unreg(inp)
73 + assert torch.allclose(out, inp * world_size)
74 +
75 +
76 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
77 + reason="Need at least 2 GPUs to run the test.")
78 +@pytest.mark.parametrize("tensor_parallel_size", [2])
79 +@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
80 +def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
81 + multi_process_tensor_parallel(tensor_parallel_size, test_target)
82 +
83 +
84 +if __name__ == "__main__":
85 + multi_process_tensor_parallel(2, graph_allreduce)
vllm/config.py
+14
@@ -328,6 +328,8 @@ class ParallelConfig:
328 worker_use_ray: Whether to use Ray for model workers. Will be set to
329 True if either pipeline_parallel_size or tensor_parallel_size is
330 greater than 1.
331 + disable_custom_all_reduce: Disable the custom all-reduce kernel and
332 + fall back to NCCL.
333 """
334
335 def __init__(
@@ -336,11 +338,13 @@ class ParallelConfig:
338 tensor_parallel_size: int,
339 worker_use_ray: bool,
340 max_parallel_loading_workers: Optional[int] = None,
341 + disable_custom_all_reduce: bool = False,
342 ) -> None:
343 self.pipeline_parallel_size = pipeline_parallel_size
344 self.tensor_parallel_size = tensor_parallel_size
345 self.worker_use_ray = worker_use_ray
346 self.max_parallel_loading_workers = max_parallel_loading_workers
347 + self.disable_custom_all_reduce = disable_custom_all_reduce
348
349 self.world_size = pipeline_parallel_size * tensor_parallel_size
350 if self.world_size > 1:
@@ -351,6 +355,16 @@ class ParallelConfig:
355 if self.pipeline_parallel_size > 1:
356 raise NotImplementedError(
357 "Pipeline parallelism is not supported yet.")
358 + if is_hip():
359 + self.disable_custom_all_reduce = True
360 + logger.info(
361 + "Disabled the custom all-reduce kernel because it is not "
362 + "supported on AMD GPUs.")
363 + elif self.pipeline_parallel_size > 1:
364 + self.disable_custom_all_reduce = True
365 + logger.info(
366 + "Disabled the custom all-reduce kernel because it is not "
367 + "supported with pipeline parallelism.")
368
369
370 class SchedulerConfig:
vllm/engine/arg_utils.py
+7 -1
@@ -35,6 +35,7 @@ class EngineArgs:
35 quantization: Optional[str] = None
36 enforce_eager: bool = False
37 max_context_len_to_capture: int = 8192
38 + disable_custom_all_reduce: bool = False
39 enable_lora: bool = False
40 max_loras: int = 1
41 max_lora_rank: int = 16
@@ -208,6 +209,10 @@ class EngineArgs:
209 help='maximum context length covered by CUDA '
210 'graphs. When a sequence has context length '
211 'larger than this, we fall back to eager mode.')
212 + parser.add_argument('--disable-custom-all-reduce',
213 + action='store_true',
214 + default=EngineArgs.disable_custom_all_reduce,
215 + help='See ParallelConfig')
216 # LoRA related configs
217 parser.add_argument('--enable-lora',
218 action='store_true',
@@ -269,7 +274,8 @@ class EngineArgs:
274 parallel_config = ParallelConfig(self.pipeline_parallel_size,
275 self.tensor_parallel_size,
276 self.worker_use_ray,
272 - self.max_parallel_loading_workers)
277 + self.max_parallel_loading_workers,
278 + self.disable_custom_all_reduce)
279 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
280 self.max_num_seqs,
281 model_config.max_model_len,
vllm/engine/llm_engine.py
+1
@@ -82,6 +82,7 @@ class LLMEngine:
82 f"download_dir={model_config.download_dir!r}, "
83 f"load_format={model_config.load_format}, "
84 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
85 + f"disable_custom_all_reduce={parallel_config.disable_custom_all_reduce}, "
86 f"quantization={model_config.quantization}, "
87 f"enforce_eager={model_config.enforce_eager}, "
88 f"seed={model_config.seed})")
vllm/entrypoints/llm.py
+3
@@ -64,6 +64,7 @@ class LLM:
64 max_context_len_to_capture: Maximum context len covered by CUDA graphs.
65 When a sequence has context length larger than this, we fall back
66 to eager mode.
67 + disable_custom_all_reduce: See ParallelConfig
68 """
69
70 def __init__(
@@ -82,6 +83,7 @@ class LLM:
83 swap_space: int = 4,
84 enforce_eager: bool = False,
85 max_context_len_to_capture: int = 8192,
86 + disable_custom_all_reduce: bool = False,
87 **kwargs,
88 ) -> None:
89 if "disable_log_stats" not in kwargs:
@@ -101,6 +103,7 @@ class LLM:
103 swap_space=swap_space,
104 enforce_eager=enforce_eager,
105 max_context_len_to_capture=max_context_len_to_capture,
106 + disable_custom_all_reduce=disable_custom_all_reduce,
107 **kwargs,
108 )
109 self.llm_engine = LLMEngine.from_engine_args(engine_args)
vllm/model_executor/parallel_utils/communication_op.py
+12 -2
@@ -10,17 +10,27 @@ from vllm.model_executor.parallel_utils.parallel_state import (
10 get_tensor_model_parallel_world_size,
11 get_tensor_model_parallel_group,
12 )
13 +from vllm.model_executor.parallel_utils.custom_all_reduce import custom_all_reduce
14
15
16 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
17 """All-reduce the input tensor across model parallel group.
18
18 - NOTE: This operation is applied in-place on the input tensor.
19 + NOTE: This operation will be applied in-place on the input tensor if
20 + disable_custom_all_reduce is set to True. Otherwise, this operation may or
21 + may not be applied in place depending on whether custom all reduce is
22 + invoked for a particular tensor, which further depends on the tensor size
23 + and GPU topology.
24 +
25 + TLDR: always assume this function modifies its input, but use the return
26 + value as the output.
27 """
28 # Bypass the function if we are using only 1 GPU.
29 if get_tensor_model_parallel_world_size() == 1:
30 return input_
23 - # All-reduce.
31 + out = custom_all_reduce(input_)
32 + if out is not None:
33 + return out
34 torch.distributed.all_reduce(input_,
35 group=get_tensor_model_parallel_group())
36 return input_
vllm/model_executor/parallel_utils/custom_all_reduce.py new
+223
@@ -0,0 +1,223 @@
1 +from contextlib import contextmanager
2 +from typing import Optional
3 +
4 +import torch
5 +import torch.distributed as dist
6 +
7 +from vllm.logger import init_logger
8 +from vllm.model_executor.parallel_utils.parallel_state import (
9 + get_tensor_model_parallel_world_size, get_tensor_model_parallel_rank)
10 +
11 +try:
12 + from vllm._C import custom_ar
13 + import pynvml
14 +except ImportError:
15 + # For AMD GPUs
16 + custom_ar = None
17 + pynvml = None
18 +
19 +logger = init_logger(__name__)
20 +
21 +_CA_HANDLE = None
22 +_IS_CAPTURING = False
23 +_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
24 +
25 +
26 +def init_custom_ar() -> None:
27 + global _CA_HANDLE
28 + if _CA_HANDLE is not None:
29 + return
30 + rank = get_tensor_model_parallel_rank()
31 + world_size = get_tensor_model_parallel_world_size()
32 + if world_size not in _SUPPORTED_WORLD_SIZES:
33 + logger.warn(
34 + "Custom allreduce is disabled due to an unsupported world size: "
35 + "%d. Supported world sizes: %s. To slience this warning, specify"
36 + "disable_custom_all_reduce=True explicitly.", world_size,
37 + str(_SUPPORTED_WORLD_SIZES))
38 + return
39 + if not _can_p2p(rank, world_size):
40 + logger.warn(
41 + "Custom allreduce is disabled because your platform lacks GPU P2P"
42 + " capability. To slience this warning, specify"
43 + "disable_custom_all_reduce=True explicitly.")
44 + return
45 + _CA_HANDLE = CustomAllreduce(rank, world_size)
46 +
47 +
48 +def begin_capture() -> None:
49 + global _IS_CAPTURING
50 + _IS_CAPTURING = True
51 +
52 +
53 +def end_capture() -> None:
54 + global _IS_CAPTURING
55 + _IS_CAPTURING = False
56 +
57 +
58 +def is_capturing() -> bool:
59 + return _IS_CAPTURING and _CA_HANDLE is not None
60 +
61 +
62 +def get_handle() -> Optional["CustomAllreduce"]:
63 + return _CA_HANDLE
64 +
65 +
66 +@contextmanager
67 +def capture():
68 + try:
69 + begin_capture()
70 + yield
71 + finally:
72 + end_capture()
73 + handle = get_handle()
74 + if handle is not None:
75 + handle.register_graph_buffers()
76 +
77 +
78 +def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
79 + ca_handle = get_handle()
80 + # when custom allreduce is disabled, this will be None
81 + if ca_handle is None:
82 + return
83 + if is_capturing():
84 + if torch.cuda.is_current_stream_capturing():
85 + if ca_handle.should_custom_ar(input):
86 + return ca_handle.all_reduce_reg(input)
87 + else:
88 + if ca_handle.should_custom_ar(input):
89 + # if warm up, mimic the allocation pattern
90 + # since custom allreduce is out-of-place
91 + return torch.empty_like(input)
92 + else:
93 + # note: outside of cuda graph context,
94 + # custom allreduce incurs a cost of cudaMemcpy, which should
95 + # be small(<=1% of overall latency) compared to the performance
96 + # gains of using custom kernels
97 + if ca_handle.should_custom_ar(input):
98 + return ca_handle.all_reduce_unreg(input)
99 +
100 +
101 +@contextmanager
102 +def _nvml():
103 + try:
104 + pynvml.nvmlInit()
105 + yield
106 + finally:
107 + pynvml.nvmlShutdown()
108 +
109 +
110 +# query if the set of gpus are fully connected by nvlink (1 hop)
111 +@_nvml()
112 +def _is_full_nvlink(rank, world_size):
113 + handle = pynvml.nvmlDeviceGetHandleByIndex(rank)
114 + for i in range(world_size):
115 + if i != rank:
116 + try:
117 + link_state = pynvml.nvmlDeviceGetNvLinkState(handle, i)
118 + if not link_state:
119 + return False
120 + except pynvml.NVMLError as error:
121 + logger.info(
122 + f"NVLink detection failed with message \"{str(error)}\". "
123 + "This is normal if your machine has no NVLink equipped")
124 + return False
125 + return True
126 +
127 +
128 +def _can_p2p(rank: int, world_size: int) -> bool:
129 + for i in range(world_size):
130 + if i == rank:
131 + continue
132 + if not torch.cuda.can_device_access_peer(rank, i):
133 + return False
134 + return True
135 +
136 +
137 +class CustomAllreduce:
138 +
139 + # max_size: max supported allreduce size
140 + def __init__(self, rank, world_size, max_size=8192 * 1024) -> None:
141 + # buffers memory are owned by this Python class and passed to C++
142 + # meta data composes of two parts: meta data for synchronization
143 + # (256 bytes) and a temporary buffer for storing intermediate
144 + # allreduce results.
145 + self.meta = torch.zeros(custom_ar.meta_size() + max_size,
146 + dtype=torch.uint8,
147 + device="cuda")
148 + # This is a pre-registered IPC buffer. In eager mode, input tensors
149 + # are first copied into this buffer before allreduce is performed
150 + self.buffer = torch.empty(max_size, dtype=torch.uint8, device="cuda")
151 + # This is a buffer for storing the tuples of pointers pointing to
152 + # IPC buffers from all ranks. Each registered tuple has size of
153 + # 8*world_size bytes where world_size is at most 8. Allocating 8MB
154 + # is enough for 131072 such tuples. The largest model I've seen only
155 + # needs less than 10000 of registered tuples.
156 + self.rank_data = torch.empty(8 * 1024 * 1024,
157 + dtype=torch.uint8,
158 + device="cuda")
159 + self.max_size = max_size
160 + self.world_size = world_size
161 + handles, offsets = self._get_ipc_meta(self.meta)
162 + self.full_nvlink = _is_full_nvlink(rank, world_size)
163 + self._ptr = custom_ar.init_custom_ar(self.meta, self.rank_data,
164 + handles, offsets, rank,
165 + self.full_nvlink)
166 + self.fast_cond = self.full_nvlink or world_size <= 2
167 + self.register_buffer(self.buffer)
168 +
169 + def _get_ipc_meta(self, inp: torch.Tensor):
170 + data = inp.untyped_storage()._share_cuda_()
171 + shard_data = (
172 + data[1], # ipc handle to base ptr
173 + data[3], # offset of base ptr
174 + )
175 + return self._gather_ipc_meta(shard_data)
176 +
177 + def _gather_ipc_meta(self, shard_data):
178 + all_data = [None] * self.world_size
179 + dist.all_gather_object(all_data, shard_data)
180 +
181 + handles = []
182 + offsets = []
183 + for i in range(len(all_data)):
184 + handles.append(all_data[i][0])
185 + offsets.append(all_data[i][1])
186 + return handles, offsets
187 +
188 + def register_buffer(self, inp: torch.Tensor):
189 + handles, offsets = self._get_ipc_meta(inp)
190 + custom_ar.register_buffer(self._ptr, inp, handles, offsets)
191 +
192 + def register_graph_buffers(self):
193 + handle, offset = custom_ar.get_graph_buffer_ipc_meta(self._ptr)
194 + handles, offsets = self._gather_ipc_meta((bytes(handle), offset))
195 + logger.info("Registering %d cuda graph addresses", len(offset))
196 + custom_ar.register_graph_buffers(self._ptr, handles, offsets)
197 +
198 + def should_custom_ar(self, inp: torch.Tensor):
199 + return custom_ar.should_custom_ar(inp, self.max_size, self.world_size,
200 + self.full_nvlink)
201 +
202 + # all reduce, assuming inp tensor is IPC registered with register_buffer,
203 + # or, in the context of cuda graphs, register_graph_buffers
204 + def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None):
205 + if out is None:
206 + out = torch.empty_like(inp)
207 + custom_ar.all_reduce_reg(self._ptr, inp, out)
208 + return out
209 +
210 + # all reduce, assuming inp tensor is NOT IPC registered
211 + def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None):
212 + if out is None:
213 + out = torch.empty_like(inp)
214 + custom_ar.all_reduce_unreg(self._ptr, inp, self.buffer, out)
215 + return out
216 +
217 + def close(self):
218 + if self._ptr:
219 + custom_ar.dispose(self._ptr)
220 + self._ptr = 0
221 +
222 + def __del__(self):
223 + self.close()
vllm/test_utils.py new
+38
@@ -0,0 +1,38 @@
1 +import ray
2 +
3 +from vllm.config import ParallelConfig
4 +from vllm.utils import get_open_port
5 +from vllm.worker.worker import init_distributed_environment
6 +
7 +
8 +def init_test_distributed_environment(
9 + pipeline_parallel_size: int,
10 + tensor_parallel_size: int,
11 + rank: int,
12 + distributed_init_port: str,
13 +) -> None:
14 + parallel_config = ParallelConfig(pipeline_parallel_size,
15 + tensor_parallel_size,
16 + worker_use_ray=True)
17 + distributed_init_method = f"tcp://localhost:{distributed_init_port}"
18 + init_distributed_environment(parallel_config, rank,
19 + distributed_init_method)
20 +
21 +
22 +def multi_process_tensor_parallel(
23 + tensor_parallel_size: int,
24 + test_target,
25 +) -> None:
26 + # Using ray helps debugging the error when it failed
27 + # as compared to multiprocessing.
28 + ray.init()
29 +
30 + distributed_init_port = get_open_port()
31 + refs = []
32 + for rank in range(tensor_parallel_size):
33 + refs.append(
34 + test_target.remote(tensor_parallel_size, rank,
35 + distributed_init_port))
36 + ray.get(refs)
37 +
38 + ray.shutdown()
vllm/worker/model_runner.py
+31 -29
@@ -10,6 +10,7 @@ from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 from vllm.model_executor.parallel_utils.communication_op import (
12 broadcast_tensor_dict)
13 +from vllm.model_executor.parallel_utils import custom_all_reduce
14 from vllm.sampling_params import SamplingParams, SamplingType
15 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
16 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
@@ -651,37 +652,38 @@ class ModelRunner:
652
653 # NOTE: Capturing the largest batch size first may help reduce the
654 # memory usage of CUDA graph.
654 - for batch_size in reversed(batch_size_capture_list):
655 - # Create dummy input_metadata.
656 - input_metadata = InputMetadata(
657 - is_prompt=False,
658 - slot_mapping=slot_mapping[:batch_size],
659 - prompt_lens=None,
660 - max_seq_len=None,
661 - start_loc=None,
662 - max_context_len=self.max_context_len_to_capture,
663 - context_lens=context_lens[:batch_size],
664 - block_tables=block_tables[:batch_size],
665 - use_cuda_graph=True,
666 - )
667 -
668 - if self.lora_config:
669 - lora_mapping = LoRAMapping(
670 - [0] * batch_size,
671 - [0] * batch_size,
655 + with custom_all_reduce.capture():
656 + for batch_size in reversed(batch_size_capture_list):
657 + # Create dummy input_metadata.
658 + input_metadata = InputMetadata(
659 + is_prompt=False,
660 + slot_mapping=slot_mapping[:batch_size],
661 + prompt_lens=None,
662 + max_seq_len=None,
663 + start_loc=None,
664 + max_context_len=self.max_context_len_to_capture,
665 + context_lens=context_lens[:batch_size],
666 + block_tables=block_tables[:batch_size],
667 + use_cuda_graph=True,
668 )
673 - self.set_active_loras(set(), lora_mapping)
669
675 - graph_runner = CUDAGraphRunner(self.model)
676 - graph_runner.capture(
677 - input_tokens[:batch_size],
678 - input_positions[:batch_size],
679 - kv_caches,
680 - input_metadata,
681 - memory_pool=self.graph_memory_pool,
682 - )
683 - self.graph_memory_pool = graph_runner.graph.pool()
684 - self.graph_runners[batch_size] = graph_runner
670 + if self.lora_config:
671 + lora_mapping = LoRAMapping(
672 + [0] * batch_size,
673 + [0] * batch_size,
674 + )
675 + self.set_active_loras(set(), lora_mapping)
676 +
677 + graph_runner = CUDAGraphRunner(self.model)
678 + graph_runner.capture(
679 + input_tokens[:batch_size],
680 + input_positions[:batch_size],
681 + kv_caches,
682 + input_metadata,
683 + memory_pool=self.graph_memory_pool,
684 + )
685 + self.graph_memory_pool = graph_runner.graph.pool()
686 + self.graph_runners[batch_size] = graph_runner
687
688 end_time = time.perf_counter()
689 elapsed_time = end_time - start_time
vllm/worker/worker.py
+6 -4
@@ -11,6 +11,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
11 from vllm.model_executor import set_random_seed
12 from vllm.model_executor.parallel_utils.communication_op import (
13 broadcast_tensor_dict)
14 +from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
15 from vllm.model_executor.parallel_utils.parallel_state import (
16 ensure_model_parallel_initialized)
17 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -78,9 +79,10 @@ class Worker:
79 _check_if_gpu_supports_dtype(self.model_config.dtype)
80
81 # Initialize the distributed environment.
81 - _init_distributed_environment(self.parallel_config, self.rank,
82 - self.distributed_init_method)
83 -
82 + init_distributed_environment(self.parallel_config, self.rank,
83 + self.distributed_init_method)
84 + if not self.parallel_config.disable_custom_all_reduce:
85 + init_custom_ar()
86 # Initialize the model.
87 set_random_seed(self.model_config.seed)
88
@@ -219,7 +221,7 @@ class Worker:
221 return self.model_runner.list_loras()
222
223
222 -def _init_distributed_environment(
224 +def init_distributed_environment(
225 parallel_config: ParallelConfig,
226 rank: int,
227 distributed_init_method: Optional[str] = None,