A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[core][distributed] use tcp store directly (#10275)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 12, 2024 at 17:36 UTC
0d4ea3fb5c8c499b70cea8b1deee3e34a147cff1
2 files changed
+29
-25
tests/distributed/test_utils.py
+16
-10
@@ -43,12 +43,15 @@ def test_cuda_device_count_stateless():
43
44
45
def cpu_worker(rank, WORLD_SIZE, port1, port2):
46
- pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
46
+ pg1 = StatelessProcessGroup.create(host="127.0.0.1",
47
+ port=port1,
48
rank=rank,
49
world_size=WORLD_SIZE)
50
if rank <= 2:
50
- pg2 = StatelessProcessGroup.create(
51
- init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
51
+ pg2 = StatelessProcessGroup.create(host="127.0.0.1",
52
+ port=port2,
53
+ rank=rank,
54
+ world_size=3)
55
data = torch.tensor([rank])
56
data = pg1.broadcast_obj(data, src=2)
57
assert data.item() == 2
@@ -62,14 +65,17 @@ def cpu_worker(rank, WORLD_SIZE, port1, port2):
65
66
def gpu_worker(rank, WORLD_SIZE, port1, port2):
67
torch.cuda.set_device(rank)
65
- pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
68
+ pg1 = StatelessProcessGroup.create(host="127.0.0.1",
69
+ port=port1,
70
rank=rank,
71
world_size=WORLD_SIZE)
72
pynccl1 = PyNcclCommunicator(pg1, device=rank)
73
pynccl1.disabled = False
74
if rank <= 2:
71
- pg2 = StatelessProcessGroup.create(
72
- init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
75
+ pg2 = StatelessProcessGroup.create(host="127.0.0.1",
76
+ port=port2,
77
+ rank=rank,
78
+ world_size=3)
79
pynccl2 = PyNcclCommunicator(pg2, device=rank)
80
pynccl2.disabled = False
81
data = torch.tensor([rank]).cuda()
@@ -89,7 +95,8 @@ def gpu_worker(rank, WORLD_SIZE, port1, port2):
95
96
97
def broadcast_worker(rank, WORLD_SIZE, port1, port2):
92
- pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
98
+ pg1 = StatelessProcessGroup.create(host="127.0.0.1",
99
+ port=port1,
100
rank=rank,
101
world_size=WORLD_SIZE)
102
if rank == 2:
@@ -101,7 +108,8 @@ def broadcast_worker(rank, WORLD_SIZE, port1, port2):
108
109
110
def allgather_worker(rank, WORLD_SIZE, port1, port2):
104
- pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
111
+ pg1 = StatelessProcessGroup.create(host="127.0.0.1",
112
+ port=port1,
113
rank=rank,
114
world_size=WORLD_SIZE)
115
data = pg1.all_gather_obj(rank)
@@ -109,8 +117,6 @@ def allgather_worker(rank, WORLD_SIZE, port1, port2):
117
pg1.barrier()
118
119
112
-# TODO: investigate why this test is flaky. It hangs during initialization.
113
-@pytest.mark.skip("Skip the test because it is flaky.")
120
@multi_gpu_test(num_gpus=4)
121
@pytest.mark.parametrize(
122
"worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker])
vllm/distributed/utils.py
+13
-15
@@ -9,7 +9,7 @@ from collections import deque
9
from typing import Any, Deque, Dict, Optional, Sequence, Tuple
10
11
import torch
12
-from torch.distributed.rendezvous import rendezvous
12
+from torch.distributed import TCPStore
13
14
import vllm.envs as envs
15
from vllm.logger import init_logger
@@ -97,7 +97,6 @@ class StatelessProcessGroup:
97
group. Only use it to communicate metadata between processes.
98
For data-plane communication, create NCCL-related objects.
99
"""
100
- prefix: str
100
rank: int
101
world_size: int
102
store: torch._C._distributed_c10d.Store
@@ -127,7 +126,7 @@ class StatelessProcessGroup:
126
def send_obj(self, obj: Any, dst: int):
127
"""Send an object to a destination rank."""
128
self.expire_data()
130
- key = f"{self.prefix}/send_to/{dst}/{self.send_dst_counter[dst]}"
129
+ key = f"send_to/{dst}/{self.send_dst_counter[dst]}"
130
self.store.set(key, pickle.dumps(obj))
131
self.send_dst_counter[dst] += 1
132
self.entries.append((key, time.time()))
@@ -147,8 +146,7 @@ class StatelessProcessGroup:
146
"""Receive an object from a source rank."""
147
obj = pickle.loads(
148
self.store.get(
150
- f"{self.prefix}/send_to/{self.rank}/{self.recv_src_counter[src]}"
151
- ))
149
+ f"send_to/{self.rank}/{self.recv_src_counter[src]}"))
150
self.recv_src_counter[src] += 1
151
return obj
152
@@ -159,14 +157,14 @@ class StatelessProcessGroup:
157
"""
158
if self.rank == src:
159
self.expire_data()
162
- key = (f"{self.prefix}/broadcast_from/{src}/"
160
+ key = (f"broadcast_from/{src}/"
161
f"{self.broadcast_send_counter}")
162
self.store.set(key, pickle.dumps(obj))
163
self.broadcast_send_counter += 1
164
self.entries.append((key, time.time()))
165
return obj
166
else:
169
- key = (f"{self.prefix}/broadcast_from/{src}/"
167
+ key = (f"broadcast_from/{src}/"
168
f"{self.broadcast_recv_src_counter[src]}")
169
recv_obj = pickle.loads(self.store.get(key))
170
self.broadcast_recv_src_counter[src] += 1
@@ -194,7 +192,8 @@ class StatelessProcessGroup:
192
193
@staticmethod
194
def create(
197
- init_method: str,
195
+ host: str,
196
+ port: int,
197
rank: int,
198
world_size: int,
199
data_expiration_seconds: int = 3600,
@@ -214,15 +213,14 @@ class StatelessProcessGroup:
213
can call `StatelessProcessGroup.create` to form a group, and then process A, B,
214
C, and D can call `StatelessProcessGroup.create` to form another group.
215
""" # noqa
217
- from torch._C._distributed_c10d import _DEFAULT_PG_TIMEOUT
218
- timeout = _DEFAULT_PG_TIMEOUT
219
-
220
- store, rank, world_size = next(
221
- rendezvous(init_method, rank, world_size, timeout=timeout))
222
- store.set_timeout(timeout)
216
+ store = TCPStore(
217
+ host_name=host,
218
+ port=port,
219
+ world_size=world_size,
220
+ is_master=(rank == 0),
221
+ )
222
223
return StatelessProcessGroup(
225
- prefix=init_method,
224
rank=rank,
225
world_size=world_size,
226
store=store,