A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[core][distributed] initialization from StatelessProcessGroup (#10986)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 12, 2024 at 01:04 UTC
62de37a38ed4a3877f3b1607b7163135f7ab9e36
5 files changed
+153
-69
.buildkite/test-pipeline.yaml
+3
-3
@@ -432,11 +432,11 @@ steps:
432
- tests/distributed/
433
commands:
434
- # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
435
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
435
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
436
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
437
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
438
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
439
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
439
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
440
441
- label: Distributed Tests (2 GPUs) # 40min
442
#mirror_hardwares: [amd]
@@ -455,7 +455,7 @@ steps:
455
commands:
456
- pytest -v -s ./compile/test_basic_correctness.py
457
- pytest -v -s ./compile/test_wrapper.py
458
- - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
458
+ - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
459
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
460
# Avoid importing model tests that cause CUDA reinitialization error
461
- pytest models/encoder_decoder/language/test_bart.py -v -s -m 'distributed(num_gpus=2)'
tests/distributed/test_same_node.py
+25
-4
@@ -3,11 +3,32 @@ import os
3
import torch.distributed as dist
4
5
from vllm.distributed.parallel_state import in_the_same_node_as
6
+from vllm.distributed.utils import StatelessProcessGroup
7
+from vllm.utils import get_ip, get_open_port
8
9
if __name__ == "__main__":
10
dist.init_process_group(backend="gloo")
9
- test_result = all(in_the_same_node_as(dist.group.WORLD, source_rank=0))
11
11
- expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12
- assert test_result == expected, f"Expected {expected}, got {test_result}"
13
- print("Same node test passed!")
12
+ rank = dist.get_rank()
13
+ if rank == 0:
14
+ port = get_open_port()
15
+ ip = get_ip()
16
+ dist.broadcast_object_list([ip, port], src=0)
17
+ else:
18
+ recv = [None, None]
19
+ dist.broadcast_object_list(recv, src=0)
20
+ ip, port = recv
21
+
22
+ stateless_pg = StatelessProcessGroup.create(ip, port, rank,
23
+ dist.get_world_size())
24
+
25
+ for pg in [dist.group.WORLD, stateless_pg]:
26
+ test_result = all(in_the_same_node_as(pg, source_rank=0))
27
+
28
+ expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
29
+ assert test_result == expected, \
30
+ f"Expected {expected}, got {test_result}"
31
+ if pg == dist.group.WORLD:
32
+ print("Same node test passed! when using torch distributed!")
33
+ else:
34
+ print("Same node test passed! when using StatelessProcessGroup!")
tests/distributed/test_shm_broadcast.py
+56
-28
@@ -7,7 +7,8 @@ import numpy as np
7
import torch.distributed as dist
8
9
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
10
-from vllm.utils import update_environment_variables
10
+from vllm.distributed.utils import StatelessProcessGroup
11
+from vllm.utils import get_ip, get_open_port, update_environment_variables
12
13
14
def get_arrays(n: int, seed: int = 0) -> List[np.ndarray]:
@@ -54,34 +55,61 @@ def worker_fn_wrapper(fn):
55
56
@worker_fn_wrapper
57
def worker_fn():
57
- writer_rank = 2
58
- broadcaster = MessageQueue.create_from_process_group(
59
- dist.group.WORLD, 40 * 1024, 2, writer_rank)
60
- if dist.get_rank() == writer_rank:
61
- seed = random.randint(0, 1000)
62
- dist.broadcast_object_list([seed], writer_rank)
63
- else:
64
- recv = [None]
65
- dist.broadcast_object_list(recv, writer_rank)
66
- seed = recv[0] # type: ignore
67
- dist.barrier()
68
- # in case we find a race condition
69
- # print the seed so that we can reproduce the error
70
- print(f"Rank {dist.get_rank()} got seed {seed}")
71
- # test broadcasting with about 400MB of data
72
- N = 10_000
73
- if dist.get_rank() == writer_rank:
74
- arrs = get_arrays(N, seed)
75
- for x in arrs:
76
- broadcaster.broadcast_object(x)
77
- time.sleep(random.random() / 1000)
58
+
59
+ rank = dist.get_rank()
60
+ if rank == 0:
61
+ port = get_open_port()
62
+ ip = get_ip()
63
+ dist.broadcast_object_list([ip, port], src=0)
64
else:
79
- arrs = get_arrays(N, seed)
80
- for x in arrs:
81
- y = broadcaster.broadcast_object(None)
82
- assert np.array_equal(x, y)
83
- time.sleep(random.random() / 1000)
84
- dist.barrier()
65
+ recv = [None, None]
66
+ dist.broadcast_object_list(recv, src=0)
67
+ ip, port = recv
68
+
69
+ stateless_pg = StatelessProcessGroup.create(ip, port, rank,
70
+ dist.get_world_size())
71
+
72
+ for pg in [dist.group.WORLD, stateless_pg]:
73
+
74
+ writer_rank = 2
75
+ broadcaster = MessageQueue.create_from_process_group(
76
+ pg, 40 * 1024, 2, writer_rank)
77
+ if rank == writer_rank:
78
+ seed = random.randint(0, 1000)
79
+ dist.broadcast_object_list([seed], writer_rank)
80
+ else:
81
+ recv = [None]
82
+ dist.broadcast_object_list(recv, writer_rank)
83
+ seed = recv[0] # type: ignore
84
+
85
+ if pg == dist.group.WORLD:
86
+ dist.barrier()
87
+ else:
88
+ pg.barrier()
89
+
90
+ # in case we find a race condition
91
+ # print the seed so that we can reproduce the error
92
+ print(f"Rank {rank} got seed {seed}")
93
+ # test broadcasting with about 400MB of data
94
+ N = 10_000
95
+ if rank == writer_rank:
96
+ arrs = get_arrays(N, seed)
97
+ for x in arrs:
98
+ broadcaster.broadcast_object(x)
99
+ time.sleep(random.random() / 1000)
100
+ else:
101
+ arrs = get_arrays(N, seed)
102
+ for x in arrs:
103
+ y = broadcaster.broadcast_object(None)
104
+ assert np.array_equal(x, y)
105
+ time.sleep(random.random() / 1000)
106
+
107
+ if pg == dist.group.WORLD:
108
+ dist.barrier()
109
+ print("torch distributed passed the test!")
110
+ else:
111
+ pg.barrier()
112
+ print("StatelessProcessGroup passed the test!")
113
114
115
def test_shm_broadcast():
vllm/distributed/device_communicators/shm_broadcast.py
+26
-13
@@ -5,7 +5,7 @@ import time
5
from contextlib import contextmanager
6
from dataclasses import dataclass, field
7
from multiprocessing import shared_memory
8
-from typing import List, Optional, Tuple
8
+from typing import List, Optional, Tuple, Union
9
from unittest.mock import patch
10
11
import torch
@@ -15,6 +15,7 @@ from zmq import IPV6 # type: ignore
15
from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore
16
17
import vllm.envs as envs
18
+from vllm.distributed.utils import StatelessProcessGroup
19
from vllm.logger import init_logger
20
from vllm.utils import get_ip, get_open_port, is_valid_ipv6_address
21
@@ -476,13 +477,19 @@ class MessageQueue:
477
return self.dequeue()
478
479
@staticmethod
479
- def create_from_process_group(pg: ProcessGroup,
480
+ def create_from_process_group(pg: Union[ProcessGroup,
481
+ StatelessProcessGroup],
482
max_chunk_bytes,
483
max_chunks,
484
writer_rank=0) -> "MessageQueue":
483
- group_rank = dist.get_rank(pg)
484
- group_world_size = dist.get_world_size(pg)
485
- global_ranks = dist.get_process_group_ranks(pg)
485
+ if isinstance(pg, ProcessGroup):
486
+ group_rank = dist.get_rank(pg)
487
+ group_world_size = dist.get_world_size(pg)
488
+ global_ranks = dist.get_process_group_ranks(pg)
489
+ else:
490
+ group_rank = pg.rank
491
+ group_world_size = pg.world_size
492
+ global_ranks = list(range(pg.world_size))
493
494
from vllm.distributed.parallel_state import in_the_same_node_as
495
status = in_the_same_node_as(pg, source_rank=writer_rank)
@@ -500,15 +507,21 @@ class MessageQueue:
507
max_chunks=max_chunks,
508
)
509
handle = buffer_io.export_handle()
503
- dist.broadcast_object_list([handle],
504
- src=global_ranks[writer_rank],
505
- group=pg)
510
+ if isinstance(pg, ProcessGroup):
511
+ dist.broadcast_object_list([handle],
512
+ src=global_ranks[writer_rank],
513
+ group=pg)
514
+ else:
515
+ pg.broadcast_obj(handle, writer_rank)
516
else:
507
- recv = [None]
508
- dist.broadcast_object_list(recv,
509
- src=global_ranks[writer_rank],
510
- group=pg)
511
- handle = recv[0] # type: ignore
517
+ if isinstance(pg, ProcessGroup):
518
+ recv = [None]
519
+ dist.broadcast_object_list(recv,
520
+ src=global_ranks[writer_rank],
521
+ group=pg)
522
+ handle = recv[0] # type: ignore
523
+ else:
524
+ handle = pg.broadcast_obj(None, writer_rank)
525
buffer_io = MessageQueue.create_from_handle(handle, group_rank)
526
buffer_io.wait_until_ready()
527
return buffer_io
vllm/distributed/parallel_state.py
+43
-21
@@ -37,6 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37
38
import vllm.distributed.kv_transfer.kv_transfer_agent as kv_transfer
39
import vllm.envs as envs
40
+from vllm.distributed.utils import StatelessProcessGroup
41
from vllm.logger import init_logger
42
from vllm.platforms import current_platform
43
from vllm.utils import direct_register_custom_op, supports_custom_op
@@ -1191,25 +1192,31 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1192
torch.cuda.empty_cache()
1193
1194
1194
-def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1195
+def in_the_same_node_as(pg: Union[ProcessGroup, StatelessProcessGroup],
1196
+ source_rank: int = 0) -> List[bool]:
1197
"""
1198
This is a collective operation that returns if each rank is in the same node
1199
as the source rank. It tests if processes are attached to the same
1200
memory system (shared access to shared memory).
1201
"""
1200
- assert torch.distributed.get_backend(
1201
- pg) != torch.distributed.Backend.NCCL, (
1202
- "in_the_same_node_as should be tested with a non-NCCL group.")
1203
- # local rank inside the group
1204
- rank = torch.distributed.get_rank(group=pg)
1205
- world_size = torch.distributed.get_world_size(group=pg)
1202
+ if isinstance(pg, ProcessGroup):
1203
+ assert torch.distributed.get_backend(
1204
+ pg) != torch.distributed.Backend.NCCL, (
1205
+ "in_the_same_node_as should be tested with a non-NCCL group.")
1206
+ # local rank inside the group
1207
+ rank = torch.distributed.get_rank(group=pg)
1208
+ world_size = torch.distributed.get_world_size(group=pg)
1209
+
1210
+ # global ranks of the processes in the group
1211
+ ranks = torch.distributed.get_process_group_ranks(pg)
1212
+ else:
1213
+ rank = pg.rank
1214
+ world_size = pg.world_size
1215
+ ranks = list(range(world_size))
1216
1217
# local tensor in each process to store the result
1218
is_in_the_same_node = torch.tensor([0] * world_size, dtype=torch.int32)
1219
1210
- # global ranks of the processes in the group
1211
- ranks = torch.distributed.get_process_group_ranks(pg)
1212
-
1220
magic_message = b"magic_message"
1221
shm = None
1222
@@ -1219,17 +1226,21 @@ def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1226
# create a shared memory segment
1227
shm = shared_memory.SharedMemory(create=True, size=128)
1228
shm.buf[:len(magic_message)] = magic_message
1222
- torch.distributed.broadcast_object_list([shm.name],
1223
- src=ranks[source_rank],
1224
- group=pg)
1229
+ if isinstance(pg, ProcessGroup):
1230
+ torch.distributed.broadcast_object_list(
1231
+ [shm.name], src=ranks[source_rank], group=pg)
1232
+ else:
1233
+ pg.broadcast_obj(shm.name, src=source_rank)
1234
is_in_the_same_node[rank] = 1
1235
else:
1236
# try to open the shared memory segment
1228
- recv = [None]
1229
- torch.distributed.broadcast_object_list(recv,
1230
- src=ranks[source_rank],
1231
- group=pg)
1232
- name = recv[0]
1237
+ if isinstance(pg, ProcessGroup):
1238
+ recv = [None]
1239
+ torch.distributed.broadcast_object_list(
1240
+ recv, src=ranks[source_rank], group=pg)
1241
+ name = recv[0]
1242
+ else:
1243
+ name = pg.broadcast_obj(None, src=source_rank)
1244
# fix to https://stackoverflow.com/q/62748654/9191338
1245
# Python incorrectly tracks shared memory even if it is not
1246
# created by the process. The following patch is a workaround.
@@ -1244,12 +1255,23 @@ def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1255
if shm:
1256
shm.close()
1257
1247
- torch.distributed.barrier(group=pg)
1258
+ if isinstance(pg, ProcessGroup):
1259
+ torch.distributed.barrier(group=pg)
1260
+ else:
1261
+ pg.barrier()
1262
1263
# clean up the shared memory segment
1264
with contextlib.suppress(OSError):
1265
if rank == source_rank and shm:
1266
shm.unlink()
1253
- torch.distributed.all_reduce(is_in_the_same_node, group=pg)
1267
1255
- return [x == 1 for x in is_in_the_same_node.tolist()]
1268
+ if isinstance(pg, ProcessGroup):
1269
+ torch.distributed.all_reduce(is_in_the_same_node, group=pg)
1270
+ aggregated_data = is_in_the_same_node
1271
+ else:
1272
+ aggregated_data = torch.zeros_like(is_in_the_same_node)
1273
+ for i in range(world_size):
1274
+ rank_data = pg.broadcast_obj(is_in_the_same_node, src=i)
1275
+ aggregated_data += rank_data
1276
+
1277
+ return [x == 1 for x in aggregated_data.tolist()]