@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0d4ea3fb5c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[core][distributed] use tcp store directly (#10275)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 12, 2024 at 17:36 UTC 0d4ea3fb5c8c499b70cea8b1deee3e34a147cff1
2 files changed +29 -25
tests/distributed/test_utils.py
+16 -10
@@ -43,12 +43,15 @@ def test_cuda_device_count_stateless():
43
44
45 def cpu_worker(rank, WORLD_SIZE, port1, port2):
46 - pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
46 + pg1 = StatelessProcessGroup.create(host="127.0.0.1",
47 + port=port1,
48 rank=rank,
49 world_size=WORLD_SIZE)
50 if rank <= 2:
50 - pg2 = StatelessProcessGroup.create(
51 - init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
51 + pg2 = StatelessProcessGroup.create(host="127.0.0.1",
52 + port=port2,
53 + rank=rank,
54 + world_size=3)
55 data = torch.tensor([rank])
56 data = pg1.broadcast_obj(data, src=2)
57 assert data.item() == 2
@@ -62,14 +65,17 @@ def cpu_worker(rank, WORLD_SIZE, port1, port2):
65
66 def gpu_worker(rank, WORLD_SIZE, port1, port2):
67 torch.cuda.set_device(rank)
65 - pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
68 + pg1 = StatelessProcessGroup.create(host="127.0.0.1",
69 + port=port1,
70 rank=rank,
71 world_size=WORLD_SIZE)
72 pynccl1 = PyNcclCommunicator(pg1, device=rank)
73 pynccl1.disabled = False
74 if rank <= 2:
71 - pg2 = StatelessProcessGroup.create(
72 - init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
75 + pg2 = StatelessProcessGroup.create(host="127.0.0.1",
76 + port=port2,
77 + rank=rank,
78 + world_size=3)
79 pynccl2 = PyNcclCommunicator(pg2, device=rank)
80 pynccl2.disabled = False
81 data = torch.tensor([rank]).cuda()
@@ -89,7 +95,8 @@ def gpu_worker(rank, WORLD_SIZE, port1, port2):
95
96
97 def broadcast_worker(rank, WORLD_SIZE, port1, port2):
92 - pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
98 + pg1 = StatelessProcessGroup.create(host="127.0.0.1",
99 + port=port1,
100 rank=rank,
101 world_size=WORLD_SIZE)
102 if rank == 2:
@@ -101,7 +108,8 @@ def broadcast_worker(rank, WORLD_SIZE, port1, port2):
108
109
110 def allgather_worker(rank, WORLD_SIZE, port1, port2):
104 - pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
111 + pg1 = StatelessProcessGroup.create(host="127.0.0.1",
112 + port=port1,
113 rank=rank,
114 world_size=WORLD_SIZE)
115 data = pg1.all_gather_obj(rank)
@@ -109,8 +117,6 @@ def allgather_worker(rank, WORLD_SIZE, port1, port2):
117 pg1.barrier()
118
119
112 -# TODO: investigate why this test is flaky. It hangs during initialization.
113 -@pytest.mark.skip("Skip the test because it is flaky.")
120 @multi_gpu_test(num_gpus=4)
121 @pytest.mark.parametrize(
122 "worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker])
vllm/distributed/utils.py
+13 -15
@@ -9,7 +9,7 @@ from collections import deque
9 from typing import Any, Deque, Dict, Optional, Sequence, Tuple
10
11 import torch
12 -from torch.distributed.rendezvous import rendezvous
12 +from torch.distributed import TCPStore
13
14 import vllm.envs as envs
15 from vllm.logger import init_logger
@@ -97,7 +97,6 @@ class StatelessProcessGroup:
97 group. Only use it to communicate metadata between processes.
98 For data-plane communication, create NCCL-related objects.
99 """
100 - prefix: str
100 rank: int
101 world_size: int
102 store: torch._C._distributed_c10d.Store
@@ -127,7 +126,7 @@ class StatelessProcessGroup:
126 def send_obj(self, obj: Any, dst: int):
127 """Send an object to a destination rank."""
128 self.expire_data()
130 - key = f"{self.prefix}/send_to/{dst}/{self.send_dst_counter[dst]}"
129 + key = f"send_to/{dst}/{self.send_dst_counter[dst]}"
130 self.store.set(key, pickle.dumps(obj))
131 self.send_dst_counter[dst] += 1
132 self.entries.append((key, time.time()))
@@ -147,8 +146,7 @@ class StatelessProcessGroup:
146 """Receive an object from a source rank."""
147 obj = pickle.loads(
148 self.store.get(
150 - f"{self.prefix}/send_to/{self.rank}/{self.recv_src_counter[src]}"
151 - ))
149 + f"send_to/{self.rank}/{self.recv_src_counter[src]}"))
150 self.recv_src_counter[src] += 1
151 return obj
152
@@ -159,14 +157,14 @@ class StatelessProcessGroup:
157 """
158 if self.rank == src:
159 self.expire_data()
162 - key = (f"{self.prefix}/broadcast_from/{src}/"
160 + key = (f"broadcast_from/{src}/"
161 f"{self.broadcast_send_counter}")
162 self.store.set(key, pickle.dumps(obj))
163 self.broadcast_send_counter += 1
164 self.entries.append((key, time.time()))
165 return obj
166 else:
169 - key = (f"{self.prefix}/broadcast_from/{src}/"
167 + key = (f"broadcast_from/{src}/"
168 f"{self.broadcast_recv_src_counter[src]}")
169 recv_obj = pickle.loads(self.store.get(key))
170 self.broadcast_recv_src_counter[src] += 1
@@ -194,7 +192,8 @@ class StatelessProcessGroup:
192
193 @staticmethod
194 def create(
197 - init_method: str,
195 + host: str,
196 + port: int,
197 rank: int,
198 world_size: int,
199 data_expiration_seconds: int = 3600,
@@ -214,15 +213,14 @@ class StatelessProcessGroup:
213 can call `StatelessProcessGroup.create` to form a group, and then process A, B,
214 C, and D can call `StatelessProcessGroup.create` to form another group.
215 """ # noqa
217 - from torch._C._distributed_c10d import _DEFAULT_PG_TIMEOUT
218 - timeout = _DEFAULT_PG_TIMEOUT
219 -
220 - store, rank, world_size = next(
221 - rendezvous(init_method, rank, world_size, timeout=timeout))
222 - store.set_timeout(timeout)
216 + store = TCPStore(
217 + host_name=host,
218 + port=port,
219 + world_size=world_size,
220 + is_master=(rank == 0),
221 + )
222
223 return StatelessProcessGroup(
225 - prefix=init_method,
224 rank=rank,
225 world_size=world_size,
226 store=store,