A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[core][distributed] add stateless process group (#10216)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 11, 2024 at 09:02 UTC
e6de9784d26fb3b0c9a55be4ab4ea3127f1900a0
3 files changed
+217
-112
tests/distributed/test_utils.py
+52
-27
@@ -1,10 +1,10 @@
1
import pytest
2
import ray
3
import torch
4
-import torch.distributed as dist
4
5
import vllm.envs as envs
7
-from vllm.distributed.utils import stateless_init_process_group
6
+from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
7
+from vllm.distributed.utils import StatelessProcessGroup
8
from vllm.utils import (cuda_device_count_stateless,
9
update_environment_variables)
10
@@ -41,42 +41,45 @@ def test_cuda_device_count_stateless():
41
42
43
def cpu_worker(rank, WORLD_SIZE):
44
- pg1 = stateless_init_process_group(init_method="tcp://127.0.0.1:29500",
44
+ pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29500",
45
rank=rank,
46
- world_size=WORLD_SIZE,
47
- backend="gloo")
46
+ world_size=WORLD_SIZE)
47
if rank <= 2:
49
- pg2 = stateless_init_process_group(init_method="tcp://127.0.0.1:29501",
48
+ pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29501",
49
rank=rank,
51
- world_size=3,
52
- backend="gloo")
50
+ world_size=3)
51
data = torch.tensor([rank])
54
- dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg1)
52
+ data = pg1.broadcast_obj(data, src=2)
53
+ assert data.item() == 2
54
if rank <= 2:
56
- dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg2)
57
- item = data[0].item()
58
- print(f"rank: {rank}, item: {item}")
59
- if rank == 3:
60
- assert item == 6
61
- else:
62
- assert item == 18
55
+ data = torch.tensor([rank + 1])
56
+ data = pg2.broadcast_obj(data, src=2)
57
+ assert data.item() == 3
58
+ pg2.barrier()
59
+ pg1.barrier()
60
61
62
def gpu_worker(rank, WORLD_SIZE):
66
- pg1 = stateless_init_process_group(init_method="tcp://127.0.0.1:29502",
63
+ torch.cuda.set_device(rank)
64
+ pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29502",
65
rank=rank,
68
- world_size=WORLD_SIZE,
69
- backend="nccl")
66
+ world_size=WORLD_SIZE)
67
+ pynccl1 = PyNcclCommunicator(pg1, device=rank)
68
+ pynccl1.disabled = False
69
if rank <= 2:
71
- pg2 = stateless_init_process_group(init_method="tcp://127.0.0.1:29503",
70
+ pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29503",
71
rank=rank,
73
- world_size=3,
74
- backend="nccl")
75
- torch.cuda.set_device(rank)
72
+ world_size=3)
73
+ pynccl2 = PyNcclCommunicator(pg2, device=rank)
74
+ pynccl2.disabled = False
75
data = torch.tensor([rank]).cuda()
77
- dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg1)
76
+ pynccl1.all_reduce(data)
77
+ pg1.barrier()
78
+ torch.cuda.synchronize()
79
if rank <= 2:
79
- dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg2)
80
+ pynccl2.all_reduce(data)
81
+ pg2.barrier()
82
+ torch.cuda.synchronize()
83
item = data[0].item()
84
print(f"rank: {rank}, item: {item}")
85
if rank == 3:
@@ -85,9 +88,31 @@ def gpu_worker(rank, WORLD_SIZE):
88
assert item == 18
89
90
91
+def broadcast_worker(rank, WORLD_SIZE):
92
+ pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29504",
93
+ rank=rank,
94
+ world_size=WORLD_SIZE)
95
+ if rank == 2:
96
+ pg1.broadcast_obj("secret", src=2)
97
+ else:
98
+ obj = pg1.broadcast_obj(None, src=2)
99
+ assert obj == "secret"
100
+ pg1.barrier()
101
+
102
+
103
+def allgather_worker(rank, WORLD_SIZE):
104
+ pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29505",
105
+ rank=rank,
106
+ world_size=WORLD_SIZE)
107
+ data = pg1.all_gather_obj(rank)
108
+ assert data == list(range(WORLD_SIZE))
109
+ pg1.barrier()
110
+
111
+
112
@multi_gpu_test(num_gpus=4)
89
-@pytest.mark.parametrize("worker", [cpu_worker, gpu_worker])
90
-def test_stateless_init_process_group(worker):
113
+@pytest.mark.parametrize(
114
+ "worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker])
115
+def test_stateless_process_group(worker):
116
WORLD_SIZE = 4
117
from multiprocessing import get_context
118
ctx = get_context("fork")
vllm/distributed/device_communicators/pynccl.py
+24
-14
@@ -9,6 +9,7 @@ from torch.distributed import ProcessGroup, ReduceOp
9
from vllm.distributed.device_communicators.pynccl_wrapper import (
10
NCCLLibrary, buffer_type, cudaStream_t, ncclComm_t, ncclDataTypeEnum,
11
ncclRedOpTypeEnum, ncclUniqueId)
12
+from vllm.distributed.utils import StatelessProcessGroup
13
from vllm.logger import init_logger
14
15
logger = init_logger(__name__)
@@ -18,7 +19,7 @@ class PyNcclCommunicator:
19
20
def __init__(
21
self,
21
- group: ProcessGroup,
22
+ group: Union[ProcessGroup, StatelessProcessGroup],
23
device: Union[int, str, torch.device],
24
library_path: Optional[str] = None,
25
):
@@ -33,13 +34,18 @@ class PyNcclCommunicator:
34
It is the caller's responsibility to make sure each communicator
35
is bind to a unique device.
36
"""
36
- assert dist.is_initialized()
37
- assert dist.get_backend(group) != dist.Backend.NCCL, (
38
- "PyNcclCommunicator should be attached to a non-NCCL group.")
37
+ if not isinstance(group, StatelessProcessGroup):
38
+ assert dist.is_initialized()
39
+ assert dist.get_backend(group) != dist.Backend.NCCL, (
40
+ "PyNcclCommunicator should be attached to a non-NCCL group.")
41
+ # note: this rank is the rank in the group
42
+ self.rank = dist.get_rank(group)
43
+ self.world_size = dist.get_world_size(group)
44
+ else:
45
+ self.rank = group.rank
46
+ self.world_size = group.world_size
47
+
48
self.group = group
40
- # note: this rank is the rank in the group
41
- self.rank = dist.get_rank(group)
42
- self.world_size = dist.get_world_size(group)
49
50
# if world_size == 1, no need to create communicator
51
if self.world_size == 1:
@@ -68,13 +74,17 @@ class PyNcclCommunicator:
74
else:
75
# construct an empty unique id
76
self.unique_id = ncclUniqueId()
71
- tensor = torch.ByteTensor(list(self.unique_id.internal))
72
- ranks = dist.get_process_group_ranks(group)
73
- # arg `src` in `broadcast` is the global rank
74
- dist.broadcast(tensor, src=ranks[0], group=group)
75
- byte_list = tensor.tolist()
76
- for i, byte in enumerate(byte_list):
77
- self.unique_id.internal[i] = byte
77
+
78
+ if not isinstance(group, StatelessProcessGroup):
79
+ tensor = torch.ByteTensor(list(self.unique_id.internal))
80
+ ranks = dist.get_process_group_ranks(group)
81
+ # arg `src` in `broadcast` is the global rank
82
+ dist.broadcast(tensor, src=ranks[0], group=group)
83
+ byte_list = tensor.tolist()
84
+ for i, byte in enumerate(byte_list):
85
+ self.unique_id.internal[i] = byte
86
+ else:
87
+ self.unique_id = group.broadcast_obj(self.unique_id, src=0)
88
if isinstance(device, int):
89
device = torch.device(f"cuda:{device}")
90
elif isinstance(device, str):
vllm/distributed/utils.py
+141
-71
@@ -2,13 +2,13 @@
2
# Adapted from
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
-from typing import Sequence, Tuple
5
+import dataclasses
6
+import pickle
7
+import time
8
+from collections import deque
9
+from typing import Any, Deque, Dict, Optional, Sequence, Tuple
10
11
import torch
8
-from torch.distributed import ProcessGroup
9
-from torch.distributed.distributed_c10d import (Backend, PrefixStore,
10
- _get_default_timeout,
11
- is_nccl_available)
12
from torch.distributed.rendezvous import rendezvous
13
14
import vllm.envs as envs
@@ -91,69 +91,139 @@ def get_pp_indices(num_hidden_layers: int, pp_rank: int,
91
return (start_layer, end_layer)
92
93
94
-def stateless_init_process_group(init_method: str, rank: int, world_size: int,
95
- backend: str) -> ProcessGroup:
96
- """A replacement for `torch.distributed.init_process_group` that does not
97
- pollute the global state.
98
-
99
- If we have process A and process B called `torch.distributed.init_process_group`
100
- to form a group, and then we want to form another group with process A, B, C,
101
- D, it is not possible in PyTorch, because process A and process B have already
102
- formed a group, and process C and process D cannot join that group. This
103
- function is a workaround for this issue.
104
-
105
- `torch.distributed.init_process_group` is a global call, while this function
106
- is a stateless call. It will return a `ProcessGroup` object that can be used
107
- for collective communication. With this function, process A and process B
108
- can call `stateless_init_process_group` to form a group, and then process A, B,
109
- C, and D can call `stateless_init_process_group` to form another group.
110
- """ # noqa
111
-
112
- backend = Backend(backend) # it is basically string
113
- timeout = _get_default_timeout(backend)
114
-
115
- store, rank, world_size = next(
116
- rendezvous(init_method, rank, world_size, timeout=timeout))
117
- store.set_timeout(timeout)
118
-
119
- group_rank = rank
120
- group_size = world_size
121
-
122
- # Use a PrefixStore to avoid accidental overrides of keys used by
123
- # different systems (e.g. RPC) in case the store is multi-tenant.
124
- prefix_store = PrefixStore(init_method, store)
125
-
126
- pg_options = ProcessGroup.Options(backend=backend, timeout=timeout)
127
-
128
- pg: ProcessGroup = ProcessGroup(
129
- prefix_store,
130
- group_rank,
131
- group_size,
132
- pg_options,
133
- )
134
-
135
- if backend == "gloo":
136
- from torch.distributed.distributed_c10d import ProcessGroupGloo
137
- backend_class = ProcessGroupGloo(prefix_store,
138
- group_rank,
139
- group_size,
140
- timeout=timeout)
141
- backend_type = ProcessGroup.BackendType.GLOO
142
- device = torch.device("cpu")
143
- elif backend == "nccl":
144
- assert is_nccl_available()
145
- from torch.distributed.distributed_c10d import ProcessGroupNCCL
146
-
147
- backend_options = ProcessGroupNCCL.Options()
148
- backend_options._timeout = timeout
149
-
150
- backend_class = ProcessGroupNCCL(prefix_store, group_rank, group_size,
151
- backend_options)
152
- backend_type = ProcessGroup.BackendType.NCCL
153
- device = torch.device("cuda")
154
-
155
- backend_class._set_sequence_number_for_group()
156
-
157
- pg._register_backend(device, backend_type, backend_class)
158
-
159
- return pg
94
+@dataclasses.dataclass
95
+class StatelessProcessGroup:
96
+ """A dataclass to hold a metadata store, and the rank, world_size of the
97
+ group. Only use it to communicate metadata between processes.
98
+ For data-plane communication, create NCCL-related objects.
99
+ """
100
+ prefix: str
101
+ rank: int
102
+ world_size: int
103
+ store: torch._C._distributed_c10d.Store
104
+ data_expiration_seconds: int = 3600 # 1 hour
105
+
106
+ # dst rank -> counter
107
+ send_dst_counter: Dict[int, int] = dataclasses.field(default_factory=dict)
108
+ # src rank -> counter
109
+ recv_src_counter: Dict[int, int] = dataclasses.field(default_factory=dict)
110
+ broadcast_send_counter: int = 0
111
+ broadcast_recv_src_counter: Dict[int, int] = dataclasses.field(
112
+ default_factory=dict)
113
+
114
+ # A deque to store the data entries, with key and timestamp.
115
+ entries: Deque[Tuple[str,
116
+ float]] = dataclasses.field(default_factory=deque)
117
+
118
+ def __post_init__(self):
119
+ assert self.rank < self.world_size
120
+ self.send_dst_counter = {i: 0 for i in range(self.world_size)}
121
+ self.recv_src_counter = {i: 0 for i in range(self.world_size)}
122
+ self.broadcast_recv_src_counter = {
123
+ i: 0
124
+ for i in range(self.world_size)
125
+ }
126
+
127
+ def send_obj(self, obj: Any, dst: int):
128
+ """Send an object to a destination rank."""
129
+ self.expire_data()
130
+ key = f"{self.prefix}/send_to/{dst}/{self.send_dst_counter[dst]}"
131
+ self.store.set(key, pickle.dumps(obj))
132
+ self.send_dst_counter[dst] += 1
133
+ self.entries.append((key, time.time()))
134
+
135
+ def expire_data(self):
136
+ """Expire data that is older than `data_expiration_seconds` seconds."""
137
+ while self.entries:
138
+ # check the oldest entry
139
+ key, timestamp = self.entries[0]
140
+ if time.time() - timestamp > self.data_expiration_seconds:
141
+ self.store.delete_key(key)
142
+ self.entries.popleft()
143
+ else:
144
+ break
145
+
146
+ def recv_obj(self, src: int) -> Any:
147
+ """Receive an object from a source rank."""
148
+ obj = pickle.loads(
149
+ self.store.get(
150
+ f"{self.prefix}/send_to/{self.rank}/{self.recv_src_counter[src]}"
151
+ ))
152
+ self.recv_src_counter[src] += 1
153
+ return obj
154
+
155
+ def broadcast_obj(self, obj: Optional[Any], src: int) -> Any:
156
+ """Broadcast an object from a source rank to all other ranks.
157
+ It does not clean up after all ranks have received the object.
158
+ Use it for limited times, e.g., for initialization.
159
+ """
160
+ if self.rank == src:
161
+ self.expire_data()
162
+ key = (f"{self.prefix}/broadcast_from/{src}/"
163
+ f"{self.broadcast_send_counter}")
164
+ self.store.set(key, pickle.dumps(obj))
165
+ self.broadcast_send_counter += 1
166
+ self.entries.append((key, time.time()))
167
+ return obj
168
+ else:
169
+ key = (f"{self.prefix}/broadcast_from/{src}/"
170
+ f"{self.broadcast_recv_src_counter[src]}")
171
+ recv_obj = pickle.loads(self.store.get(key))
172
+ self.broadcast_recv_src_counter[src] += 1
173
+ return recv_obj
174
+
175
+ def all_gather_obj(self, obj: Any) -> list[Any]:
176
+ """All gather an object from all ranks."""
177
+ gathered_objs = []
178
+ for i in range(self.world_size):
179
+ if i == self.rank:
180
+ gathered_objs.append(obj)
181
+ self.broadcast_obj(obj, src=self.rank)
182
+ else:
183
+ recv_obj = self.broadcast_obj(None, src=i)
184
+ gathered_objs.append(recv_obj)
185
+ return gathered_objs
186
+
187
+ def barrier(self):
188
+ """A barrier to synchronize all ranks."""
189
+ for i in range(self.world_size):
190
+ if i == self.rank:
191
+ self.broadcast_obj(None, src=self.rank)
192
+ else:
193
+ self.broadcast_obj(None, src=i)
194
+
195
+ @staticmethod
196
+ def create(
197
+ init_method: str,
198
+ rank: int,
199
+ world_size: int,
200
+ data_expiration_seconds: int = 3600,
201
+ ) -> "StatelessProcessGroup":
202
+ """A replacement for `torch.distributed.init_process_group` that does not
203
+ pollute the global state.
204
+
205
+ If we have process A and process B called `torch.distributed.init_process_group`
206
+ to form a group, and then we want to form another group with process A, B, C,
207
+ D, it is not possible in PyTorch, because process A and process B have already
208
+ formed a group, and process C and process D cannot join that group. This
209
+ function is a workaround for this issue.
210
+
211
+ `torch.distributed.init_process_group` is a global call, while this function
212
+ is a stateless call. It will return a `StatelessProcessGroup` object that can be
213
+ used for exchanging metadata. With this function, process A and process B
214
+ can call `StatelessProcessGroup.create` to form a group, and then process A, B,
215
+ C, and D can call `StatelessProcessGroup.create` to form another group.
216
+ """ # noqa
217
+ from torch._C._distributed_c10d import _DEFAULT_PG_TIMEOUT
218
+ timeout = _DEFAULT_PG_TIMEOUT
219
+
220
+ store, rank, world_size = next(
221
+ rendezvous(init_method, rank, world_size, timeout=timeout))
222
+ store.set_timeout(timeout)
223
+
224
+ return StatelessProcessGroup(
225
+ prefix=init_method,
226
+ rank=rank,
227
+ world_size=world_size,
228
+ store=store,
229
+ data_expiration_seconds=data_expiration_seconds)