@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e6de9784d2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[core][distributed] add stateless process group (#10216)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 11, 2024 at 09:02 UTC e6de9784d26fb3b0c9a55be4ab4ea3127f1900a0
3 files changed +217 -112
tests/distributed/test_utils.py
+52 -27
@@ -1,10 +1,10 @@
1 import pytest
2 import ray
3 import torch
4 -import torch.distributed as dist
4
5 import vllm.envs as envs
7 -from vllm.distributed.utils import stateless_init_process_group
6 +from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
7 +from vllm.distributed.utils import StatelessProcessGroup
8 from vllm.utils import (cuda_device_count_stateless,
9 update_environment_variables)
10
@@ -41,42 +41,45 @@ def test_cuda_device_count_stateless():
41
42
43 def cpu_worker(rank, WORLD_SIZE):
44 - pg1 = stateless_init_process_group(init_method="tcp://127.0.0.1:29500",
44 + pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29500",
45 rank=rank,
46 - world_size=WORLD_SIZE,
47 - backend="gloo")
46 + world_size=WORLD_SIZE)
47 if rank <= 2:
49 - pg2 = stateless_init_process_group(init_method="tcp://127.0.0.1:29501",
48 + pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29501",
49 rank=rank,
51 - world_size=3,
52 - backend="gloo")
50 + world_size=3)
51 data = torch.tensor([rank])
54 - dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg1)
52 + data = pg1.broadcast_obj(data, src=2)
53 + assert data.item() == 2
54 if rank <= 2:
56 - dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg2)
57 - item = data[0].item()
58 - print(f"rank: {rank}, item: {item}")
59 - if rank == 3:
60 - assert item == 6
61 - else:
62 - assert item == 18
55 + data = torch.tensor([rank + 1])
56 + data = pg2.broadcast_obj(data, src=2)
57 + assert data.item() == 3
58 + pg2.barrier()
59 + pg1.barrier()
60
61
62 def gpu_worker(rank, WORLD_SIZE):
66 - pg1 = stateless_init_process_group(init_method="tcp://127.0.0.1:29502",
63 + torch.cuda.set_device(rank)
64 + pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29502",
65 rank=rank,
68 - world_size=WORLD_SIZE,
69 - backend="nccl")
66 + world_size=WORLD_SIZE)
67 + pynccl1 = PyNcclCommunicator(pg1, device=rank)
68 + pynccl1.disabled = False
69 if rank <= 2:
71 - pg2 = stateless_init_process_group(init_method="tcp://127.0.0.1:29503",
70 + pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29503",
71 rank=rank,
73 - world_size=3,
74 - backend="nccl")
75 - torch.cuda.set_device(rank)
72 + world_size=3)
73 + pynccl2 = PyNcclCommunicator(pg2, device=rank)
74 + pynccl2.disabled = False
75 data = torch.tensor([rank]).cuda()
77 - dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg1)
76 + pynccl1.all_reduce(data)
77 + pg1.barrier()
78 + torch.cuda.synchronize()
79 if rank <= 2:
79 - dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg2)
80 + pynccl2.all_reduce(data)
81 + pg2.barrier()
82 + torch.cuda.synchronize()
83 item = data[0].item()
84 print(f"rank: {rank}, item: {item}")
85 if rank == 3:
@@ -85,9 +88,31 @@ def gpu_worker(rank, WORLD_SIZE):
88 assert item == 18
89
90
91 +def broadcast_worker(rank, WORLD_SIZE):
92 + pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29504",
93 + rank=rank,
94 + world_size=WORLD_SIZE)
95 + if rank == 2:
96 + pg1.broadcast_obj("secret", src=2)
97 + else:
98 + obj = pg1.broadcast_obj(None, src=2)
99 + assert obj == "secret"
100 + pg1.barrier()
101 +
102 +
103 +def allgather_worker(rank, WORLD_SIZE):
104 + pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29505",
105 + rank=rank,
106 + world_size=WORLD_SIZE)
107 + data = pg1.all_gather_obj(rank)
108 + assert data == list(range(WORLD_SIZE))
109 + pg1.barrier()
110 +
111 +
112 @multi_gpu_test(num_gpus=4)
89 -@pytest.mark.parametrize("worker", [cpu_worker, gpu_worker])
90 -def test_stateless_init_process_group(worker):
113 +@pytest.mark.parametrize(
114 + "worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker])
115 +def test_stateless_process_group(worker):
116 WORLD_SIZE = 4
117 from multiprocessing import get_context
118 ctx = get_context("fork")
vllm/distributed/device_communicators/pynccl.py
+24 -14
@@ -9,6 +9,7 @@ from torch.distributed import ProcessGroup, ReduceOp
9 from vllm.distributed.device_communicators.pynccl_wrapper import (
10 NCCLLibrary, buffer_type, cudaStream_t, ncclComm_t, ncclDataTypeEnum,
11 ncclRedOpTypeEnum, ncclUniqueId)
12 +from vllm.distributed.utils import StatelessProcessGroup
13 from vllm.logger import init_logger
14
15 logger = init_logger(__name__)
@@ -18,7 +19,7 @@ class PyNcclCommunicator:
19
20 def __init__(
21 self,
21 - group: ProcessGroup,
22 + group: Union[ProcessGroup, StatelessProcessGroup],
23 device: Union[int, str, torch.device],
24 library_path: Optional[str] = None,
25 ):
@@ -33,13 +34,18 @@ class PyNcclCommunicator:
34 It is the caller's responsibility to make sure each communicator
35 is bind to a unique device.
36 """
36 - assert dist.is_initialized()
37 - assert dist.get_backend(group) != dist.Backend.NCCL, (
38 - "PyNcclCommunicator should be attached to a non-NCCL group.")
37 + if not isinstance(group, StatelessProcessGroup):
38 + assert dist.is_initialized()
39 + assert dist.get_backend(group) != dist.Backend.NCCL, (
40 + "PyNcclCommunicator should be attached to a non-NCCL group.")
41 + # note: this rank is the rank in the group
42 + self.rank = dist.get_rank(group)
43 + self.world_size = dist.get_world_size(group)
44 + else:
45 + self.rank = group.rank
46 + self.world_size = group.world_size
47 +
48 self.group = group
40 - # note: this rank is the rank in the group
41 - self.rank = dist.get_rank(group)
42 - self.world_size = dist.get_world_size(group)
49
50 # if world_size == 1, no need to create communicator
51 if self.world_size == 1:
@@ -68,13 +74,17 @@ class PyNcclCommunicator:
74 else:
75 # construct an empty unique id
76 self.unique_id = ncclUniqueId()
71 - tensor = torch.ByteTensor(list(self.unique_id.internal))
72 - ranks = dist.get_process_group_ranks(group)
73 - # arg `src` in `broadcast` is the global rank
74 - dist.broadcast(tensor, src=ranks[0], group=group)
75 - byte_list = tensor.tolist()
76 - for i, byte in enumerate(byte_list):
77 - self.unique_id.internal[i] = byte
77 +
78 + if not isinstance(group, StatelessProcessGroup):
79 + tensor = torch.ByteTensor(list(self.unique_id.internal))
80 + ranks = dist.get_process_group_ranks(group)
81 + # arg `src` in `broadcast` is the global rank
82 + dist.broadcast(tensor, src=ranks[0], group=group)
83 + byte_list = tensor.tolist()
84 + for i, byte in enumerate(byte_list):
85 + self.unique_id.internal[i] = byte
86 + else:
87 + self.unique_id = group.broadcast_obj(self.unique_id, src=0)
88 if isinstance(device, int):
89 device = torch.device(f"cuda:{device}")
90 elif isinstance(device, str):
vllm/distributed/utils.py
+141 -71
@@ -2,13 +2,13 @@
2 # Adapted from
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 -from typing import Sequence, Tuple
5 +import dataclasses
6 +import pickle
7 +import time
8 +from collections import deque
9 +from typing import Any, Deque, Dict, Optional, Sequence, Tuple
10
11 import torch
8 -from torch.distributed import ProcessGroup
9 -from torch.distributed.distributed_c10d import (Backend, PrefixStore,
10 - _get_default_timeout,
11 - is_nccl_available)
12 from torch.distributed.rendezvous import rendezvous
13
14 import vllm.envs as envs
@@ -91,69 +91,139 @@ def get_pp_indices(num_hidden_layers: int, pp_rank: int,
91 return (start_layer, end_layer)
92
93
94 -def stateless_init_process_group(init_method: str, rank: int, world_size: int,
95 - backend: str) -> ProcessGroup:
96 - """A replacement for `torch.distributed.init_process_group` that does not
97 - pollute the global state.
98 -
99 - If we have process A and process B called `torch.distributed.init_process_group`
100 - to form a group, and then we want to form another group with process A, B, C,
101 - D, it is not possible in PyTorch, because process A and process B have already
102 - formed a group, and process C and process D cannot join that group. This
103 - function is a workaround for this issue.
104 -
105 - `torch.distributed.init_process_group` is a global call, while this function
106 - is a stateless call. It will return a `ProcessGroup` object that can be used
107 - for collective communication. With this function, process A and process B
108 - can call `stateless_init_process_group` to form a group, and then process A, B,
109 - C, and D can call `stateless_init_process_group` to form another group.
110 - """ # noqa
111 -
112 - backend = Backend(backend) # it is basically string
113 - timeout = _get_default_timeout(backend)
114 -
115 - store, rank, world_size = next(
116 - rendezvous(init_method, rank, world_size, timeout=timeout))
117 - store.set_timeout(timeout)
118 -
119 - group_rank = rank
120 - group_size = world_size
121 -
122 - # Use a PrefixStore to avoid accidental overrides of keys used by
123 - # different systems (e.g. RPC) in case the store is multi-tenant.
124 - prefix_store = PrefixStore(init_method, store)
125 -
126 - pg_options = ProcessGroup.Options(backend=backend, timeout=timeout)
127 -
128 - pg: ProcessGroup = ProcessGroup(
129 - prefix_store,
130 - group_rank,
131 - group_size,
132 - pg_options,
133 - )
134 -
135 - if backend == "gloo":
136 - from torch.distributed.distributed_c10d import ProcessGroupGloo
137 - backend_class = ProcessGroupGloo(prefix_store,
138 - group_rank,
139 - group_size,
140 - timeout=timeout)
141 - backend_type = ProcessGroup.BackendType.GLOO
142 - device = torch.device("cpu")
143 - elif backend == "nccl":
144 - assert is_nccl_available()
145 - from torch.distributed.distributed_c10d import ProcessGroupNCCL
146 -
147 - backend_options = ProcessGroupNCCL.Options()
148 - backend_options._timeout = timeout
149 -
150 - backend_class = ProcessGroupNCCL(prefix_store, group_rank, group_size,
151 - backend_options)
152 - backend_type = ProcessGroup.BackendType.NCCL
153 - device = torch.device("cuda")
154 -
155 - backend_class._set_sequence_number_for_group()
156 -
157 - pg._register_backend(device, backend_type, backend_class)
158 -
159 - return pg
94 +@dataclasses.dataclass
95 +class StatelessProcessGroup:
96 + """A dataclass to hold a metadata store, and the rank, world_size of the
97 + group. Only use it to communicate metadata between processes.
98 + For data-plane communication, create NCCL-related objects.
99 + """
100 + prefix: str
101 + rank: int
102 + world_size: int
103 + store: torch._C._distributed_c10d.Store
104 + data_expiration_seconds: int = 3600 # 1 hour
105 +
106 + # dst rank -> counter
107 + send_dst_counter: Dict[int, int] = dataclasses.field(default_factory=dict)
108 + # src rank -> counter
109 + recv_src_counter: Dict[int, int] = dataclasses.field(default_factory=dict)
110 + broadcast_send_counter: int = 0
111 + broadcast_recv_src_counter: Dict[int, int] = dataclasses.field(
112 + default_factory=dict)
113 +
114 + # A deque to store the data entries, with key and timestamp.
115 + entries: Deque[Tuple[str,
116 + float]] = dataclasses.field(default_factory=deque)
117 +
118 + def __post_init__(self):
119 + assert self.rank < self.world_size
120 + self.send_dst_counter = {i: 0 for i in range(self.world_size)}
121 + self.recv_src_counter = {i: 0 for i in range(self.world_size)}
122 + self.broadcast_recv_src_counter = {
123 + i: 0
124 + for i in range(self.world_size)
125 + }
126 +
127 + def send_obj(self, obj: Any, dst: int):
128 + """Send an object to a destination rank."""
129 + self.expire_data()
130 + key = f"{self.prefix}/send_to/{dst}/{self.send_dst_counter[dst]}"
131 + self.store.set(key, pickle.dumps(obj))
132 + self.send_dst_counter[dst] += 1
133 + self.entries.append((key, time.time()))
134 +
135 + def expire_data(self):
136 + """Expire data that is older than `data_expiration_seconds` seconds."""
137 + while self.entries:
138 + # check the oldest entry
139 + key, timestamp = self.entries[0]
140 + if time.time() - timestamp > self.data_expiration_seconds:
141 + self.store.delete_key(key)
142 + self.entries.popleft()
143 + else:
144 + break
145 +
146 + def recv_obj(self, src: int) -> Any:
147 + """Receive an object from a source rank."""
148 + obj = pickle.loads(
149 + self.store.get(
150 + f"{self.prefix}/send_to/{self.rank}/{self.recv_src_counter[src]}"
151 + ))
152 + self.recv_src_counter[src] += 1
153 + return obj
154 +
155 + def broadcast_obj(self, obj: Optional[Any], src: int) -> Any:
156 + """Broadcast an object from a source rank to all other ranks.
157 + It does not clean up after all ranks have received the object.
158 + Use it for limited times, e.g., for initialization.
159 + """
160 + if self.rank == src:
161 + self.expire_data()
162 + key = (f"{self.prefix}/broadcast_from/{src}/"
163 + f"{self.broadcast_send_counter}")
164 + self.store.set(key, pickle.dumps(obj))
165 + self.broadcast_send_counter += 1
166 + self.entries.append((key, time.time()))
167 + return obj
168 + else:
169 + key = (f"{self.prefix}/broadcast_from/{src}/"
170 + f"{self.broadcast_recv_src_counter[src]}")
171 + recv_obj = pickle.loads(self.store.get(key))
172 + self.broadcast_recv_src_counter[src] += 1
173 + return recv_obj
174 +
175 + def all_gather_obj(self, obj: Any) -> list[Any]:
176 + """All gather an object from all ranks."""
177 + gathered_objs = []
178 + for i in range(self.world_size):
179 + if i == self.rank:
180 + gathered_objs.append(obj)
181 + self.broadcast_obj(obj, src=self.rank)
182 + else:
183 + recv_obj = self.broadcast_obj(None, src=i)
184 + gathered_objs.append(recv_obj)
185 + return gathered_objs
186 +
187 + def barrier(self):
188 + """A barrier to synchronize all ranks."""
189 + for i in range(self.world_size):
190 + if i == self.rank:
191 + self.broadcast_obj(None, src=self.rank)
192 + else:
193 + self.broadcast_obj(None, src=i)
194 +
195 + @staticmethod
196 + def create(
197 + init_method: str,
198 + rank: int,
199 + world_size: int,
200 + data_expiration_seconds: int = 3600,
201 + ) -> "StatelessProcessGroup":
202 + """A replacement for `torch.distributed.init_process_group` that does not
203 + pollute the global state.
204 +
205 + If we have process A and process B called `torch.distributed.init_process_group`
206 + to form a group, and then we want to form another group with process A, B, C,
207 + D, it is not possible in PyTorch, because process A and process B have already
208 + formed a group, and process C and process D cannot join that group. This
209 + function is a workaround for this issue.
210 +
211 + `torch.distributed.init_process_group` is a global call, while this function
212 + is a stateless call. It will return a `StatelessProcessGroup` object that can be
213 + used for exchanging metadata. With this function, process A and process B
214 + can call `StatelessProcessGroup.create` to form a group, and then process A, B,
215 + C, and D can call `StatelessProcessGroup.create` to form another group.
216 + """ # noqa
217 + from torch._C._distributed_c10d import _DEFAULT_PG_TIMEOUT
218 + timeout = _DEFAULT_PG_TIMEOUT
219 +
220 + store, rank, world_size = next(
221 + rendezvous(init_method, rank, world_size, timeout=timeout))
222 + store.set_timeout(timeout)
223 +
224 + return StatelessProcessGroup(
225 + prefix=init_method,
226 + rank=rank,
227 + world_size=world_size,
228 + store=store,
229 + data_expiration_seconds=data_expiration_seconds)