@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a7fe47d32
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[misc][distributed] auto port selection and disable tests (#10226)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 11, 2024 at 11:54 UTC 8a7fe47d322920bdff1b1c3472fe7f423a73a23b
1 file changed +23 -16
tests/distributed/test_utils.py
+23 -16
@@ -1,3 +1,5 @@
1 +import socket
2 +
3 import pytest
4 import ray
5 import torch
@@ -5,7 +7,7 @@ import torch
7 import vllm.envs as envs
8 from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
9 from vllm.distributed.utils import StatelessProcessGroup
8 -from vllm.utils import (cuda_device_count_stateless,
10 +from vllm.utils import (cuda_device_count_stateless, get_open_port,
11 update_environment_variables)
12
13 from ..utils import multi_gpu_test
@@ -40,14 +42,13 @@ def test_cuda_device_count_stateless():
42 assert ray.get(actor.get_count.remote()) == 0
43
44
43 -def cpu_worker(rank, WORLD_SIZE):
44 - pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29500",
45 +def cpu_worker(rank, WORLD_SIZE, port1, port2):
46 + pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
47 rank=rank,
48 world_size=WORLD_SIZE)
49 if rank <= 2:
48 - pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29501",
49 - rank=rank,
50 - world_size=3)
50 + pg2 = StatelessProcessGroup.create(
51 + init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
52 data = torch.tensor([rank])
53 data = pg1.broadcast_obj(data, src=2)
54 assert data.item() == 2
@@ -59,17 +60,16 @@ def cpu_worker(rank, WORLD_SIZE):
60 pg1.barrier()
61
62
62 -def gpu_worker(rank, WORLD_SIZE):
63 +def gpu_worker(rank, WORLD_SIZE, port1, port2):
64 torch.cuda.set_device(rank)
64 - pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29502",
65 + pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
66 rank=rank,
67 world_size=WORLD_SIZE)
68 pynccl1 = PyNcclCommunicator(pg1, device=rank)
69 pynccl1.disabled = False
70 if rank <= 2:
70 - pg2 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29503",
71 - rank=rank,
72 - world_size=3)
71 + pg2 = StatelessProcessGroup.create(
72 + init_method=f"tcp://127.0.0.1:{port2}", rank=rank, world_size=3)
73 pynccl2 = PyNcclCommunicator(pg2, device=rank)
74 pynccl2.disabled = False
75 data = torch.tensor([rank]).cuda()
@@ -88,8 +88,8 @@ def gpu_worker(rank, WORLD_SIZE):
88 assert item == 18
89
90
91 -def broadcast_worker(rank, WORLD_SIZE):
92 - pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29504",
91 +def broadcast_worker(rank, WORLD_SIZE, port1, port2):
92 + pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
93 rank=rank,
94 world_size=WORLD_SIZE)
95 if rank == 2:
@@ -100,8 +100,8 @@ def broadcast_worker(rank, WORLD_SIZE):
100 pg1.barrier()
101
102
103 -def allgather_worker(rank, WORLD_SIZE):
104 - pg1 = StatelessProcessGroup.create(init_method="tcp://127.0.0.1:29505",
103 +def allgather_worker(rank, WORLD_SIZE, port1, port2):
104 + pg1 = StatelessProcessGroup.create(init_method=f"tcp://127.0.0.1:{port1}",
105 rank=rank,
106 world_size=WORLD_SIZE)
107 data = pg1.all_gather_obj(rank)
@@ -109,17 +109,24 @@ def allgather_worker(rank, WORLD_SIZE):
109 pg1.barrier()
110
111
112 +# TODO: investigate why this test is flaky. It hangs during initialization.
113 +@pytest.mark.skip("Skip the test because it is flaky.")
114 @multi_gpu_test(num_gpus=4)
115 @pytest.mark.parametrize(
116 "worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker])
117 def test_stateless_process_group(worker):
118 + port1 = get_open_port()
119 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
120 + s.bind(("", port1))
121 + port2 = get_open_port()
122 WORLD_SIZE = 4
123 from multiprocessing import get_context
124 ctx = get_context("fork")
125 processes = []
126 for i in range(WORLD_SIZE):
127 rank = i
122 - processes.append(ctx.Process(target=worker, args=(rank, WORLD_SIZE)))
128 + processes.append(
129 + ctx.Process(target=worker, args=(rank, WORLD_SIZE, port1, port2)))
130 for p in processes:
131 p.start()
132 for p in processes: