A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[misc][distributed] auto port selection and disable tests (#10226)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 11, 2024 at 11:54 UTC
8a7fe47d322920bdff1b1c3472fe7f423a73a23b
1 file changed
+23
-16
tests/distributed/test_utils.py
+23
-16
@@ -1,3 +1,5 @@
1
+import socket
2
+
3
import pytest
4
import ray
5
import torch
@@ -5,7 +7,7 @@ import torch
7
import vllm.envs as envs
8
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
9
from vllm.distributed.utils import StatelessProcessGroup
8
-from vllm.utils import (cuda_device_count_stateless,
10
+from vllm.utils import (cuda_device_count_stateless, get_open_port,
11
update_environment_variables)
12
13
from ..utils import multi_gpu_test
@@ -40,14 +42,13 @@ def test_cuda_device_count_stateless():
42
assert ray.get(actor.get_count.remote()) == 0
43
44
43
-def cpu_worker(rank, WORLD_SIZE):
44
- pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29500",
45
+def cpu_worker(rank, WORLD_SIZE, port1, port2):
46
+ pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
47
rank=rank,
48
world_size=WORLD_SIZE)
49
if rank <= 2:
48
- pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29501",
49
- rank=rank,
50
- world_size=3)
50
+ pg2 = StatelessProcessGroup.create(
51
+ init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
52
data = torch.tensor([rank])
53
data = pg1.broadcast_obj(data, src=2)
54
assert data.item() == 2
@@ -59,17 +60,16 @@ def cpu_worker(rank, WORLD_SIZE):
60
pg1.barrier()
61
62
62
-def gpu_worker(rank, WORLD_SIZE):
63
+def gpu_worker(rank, WORLD_SIZE, port1, port2):
64
torch.cuda.set_device(rank)
64
- pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29502",
65
+ pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
66
rank=rank,
67
world_size=WORLD_SIZE)
68
pynccl1 = PyNcclCommunicator(pg1, device=rank)
69
pynccl1.disabled = False
70
if rank <= 2:
70
- pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29503",
71
- rank=rank,
72
- world_size=3)
71
+ pg2 = StatelessProcessGroup.create(
72
+ init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
73
pynccl2 = PyNcclCommunicator(pg2, device=rank)
74
pynccl2.disabled = False
75
data = torch.tensor([rank]).cuda()
@@ -88,8 +88,8 @@ def gpu_worker(rank, WORLD_SIZE):
88
assert item == 18
89
90
91
-def broadcast_worker(rank, WORLD_SIZE):
92
- pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29504",
91
+def broadcast_worker(rank, WORLD_SIZE, port1, port2):
92
+ pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
93
rank=rank,
94
world_size=WORLD_SIZE)
95
if rank == 2:
@@ -100,8 +100,8 @@ def broadcast_worker(rank, WORLD_SIZE):
100
pg1.barrier()
101
102
103
-def allgather_worker(rank, WORLD_SIZE):
104
- pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29505",
103
+def allgather_worker(rank, WORLD_SIZE, port1, port2):
104
+ pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
105
rank=rank,
106
world_size=WORLD_SIZE)
107
data = pg1.all_gather_obj(rank)
@@ -109,17 +109,24 @@ def allgather_worker(rank, WORLD_SIZE):
109
pg1.barrier()
110
111
112
+# TODO: investigate why this test is flaky. It hangs during initialization.
113
+@pytest.mark.skip("Skip the test because it is flaky.")
114
@multi_gpu_test(num_gpus=4)
115
@pytest.mark.parametrize(
116
"worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker])
117
def test_stateless_process_group(worker):
118
+ port1 = get_open_port()
119
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
120
+ s.bind(("", port1))
121
+ port2 = get_open_port()
122
WORLD_SIZE = 4
123
from multiprocessing import get_context
124
ctx = get_context("fork")
125
processes = []
126
for i in range(WORLD_SIZE):
127
rank = i
122
- processes.append(ctx.Process(target=worker, args=(rank, WORLD_SIZE)))
128
+ processes.append(
129
+ ctx.Process(target=worker, args=(rank, WORLD_SIZE, port1, port2)))
130
for p in processes:
131
p.start()
132
for p in processes: