@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 62de37a38e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[core][distributed] initialization from StatelessProcessGroup (#10986)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 12, 2024 at 01:04 UTC 62de37a38ed4a3877f3b1607b7163135f7ab9e36
5 files changed +153 -69
.buildkite/test-pipeline.yaml
+3 -3
@@ -432,11 +432,11 @@ steps:
432 - tests/distributed/
433 commands:
434 - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
435 - - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
435 + - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
436 - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
437 - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
438 - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
439 - - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
439 + - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
440
441 - label: Distributed Tests (2 GPUs) # 40min
442 #mirror_hardwares: [amd]
@@ -455,7 +455,7 @@ steps:
455 commands:
456 - pytest -v -s ./compile/test_basic_correctness.py
457 - pytest -v -s ./compile/test_wrapper.py
458 - - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
458 + - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
459 - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
460 # Avoid importing model tests that cause CUDA reinitialization error
461 - pytest models/encoder_decoder/language/test_bart.py -v -s -m 'distributed(num_gpus=2)'
tests/distributed/test_same_node.py
+25 -4
@@ -3,11 +3,32 @@ import os
3 import torch.distributed as dist
4
5 from vllm.distributed.parallel_state import in_the_same_node_as
6 +from vllm.distributed.utils import StatelessProcessGroup
7 +from vllm.utils import get_ip, get_open_port
8
9 if __name__ == "__main__":
10 dist.init_process_group(backend="gloo")
9 - test_result = all(in_the_same_node_as(dist.group.WORLD, source_rank=0))
11
11 - expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12 - assert test_result == expected, f"Expected {expected}, got {test_result}"
13 - print("Same node test passed!")
12 + rank = dist.get_rank()
13 + if rank == 0:
14 + port = get_open_port()
15 + ip = get_ip()
16 + dist.broadcast_object_list([ip, port], src=0)
17 + else:
18 + recv = [None, None]
19 + dist.broadcast_object_list(recv, src=0)
20 + ip, port = recv
21 +
22 + stateless_pg = StatelessProcessGroup.create(ip, port, rank,
23 + dist.get_world_size())
24 +
25 + for pg in [dist.group.WORLD, stateless_pg]:
26 + test_result = all(in_the_same_node_as(pg, source_rank=0))
27 +
28 + expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
29 + assert test_result == expected, \
30 + f"Expected {expected}, got {test_result}"
31 + if pg == dist.group.WORLD:
32 + print("Same node test passed! when using torch distributed!")
33 + else:
34 + print("Same node test passed! when using StatelessProcessGroup!")
tests/distributed/test_shm_broadcast.py
+56 -28
@@ -7,7 +7,8 @@ import numpy as np
7 import torch.distributed as dist
8
9 from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
10 -from vllm.utils import update_environment_variables
10 +from vllm.distributed.utils import StatelessProcessGroup
11 +from vllm.utils import get_ip, get_open_port, update_environment_variables
12
13
14 def get_arrays(n: int, seed: int = 0) -> List[np.ndarray]:
@@ -54,34 +55,61 @@ def worker_fn_wrapper(fn):
55
56 @worker_fn_wrapper
57 def worker_fn():
57 - writer_rank = 2
58 - broadcaster = MessageQueue.create_from_process_group(
59 - dist.group.WORLD, 40 * 1024, 2, writer_rank)
60 - if dist.get_rank() == writer_rank:
61 - seed = random.randint(0, 1000)
62 - dist.broadcast_object_list([seed], writer_rank)
63 - else:
64 - recv = [None]
65 - dist.broadcast_object_list(recv, writer_rank)
66 - seed = recv[0] # type: ignore
67 - dist.barrier()
68 - # in case we find a race condition
69 - # print the seed so that we can reproduce the error
70 - print(f"Rank {dist.get_rank()} got seed {seed}")
71 - # test broadcasting with about 400MB of data
72 - N = 10_000
73 - if dist.get_rank() == writer_rank:
74 - arrs = get_arrays(N, seed)
75 - for x in arrs:
76 - broadcaster.broadcast_object(x)
77 - time.sleep(random.random() / 1000)
58 +
59 + rank = dist.get_rank()
60 + if rank == 0:
61 + port = get_open_port()
62 + ip = get_ip()
63 + dist.broadcast_object_list([ip, port], src=0)
64 else:
79 - arrs = get_arrays(N, seed)
80 - for x in arrs:
81 - y = broadcaster.broadcast_object(None)
82 - assert np.array_equal(x, y)
83 - time.sleep(random.random() / 1000)
84 - dist.barrier()
65 + recv = [None, None]
66 + dist.broadcast_object_list(recv, src=0)
67 + ip, port = recv
68 +
69 + stateless_pg = StatelessProcessGroup.create(ip, port, rank,
70 + dist.get_world_size())
71 +
72 + for pg in [dist.group.WORLD, stateless_pg]:
73 +
74 + writer_rank = 2
75 + broadcaster = MessageQueue.create_from_process_group(
76 + pg, 40 * 1024, 2, writer_rank)
77 + if rank == writer_rank:
78 + seed = random.randint(0, 1000)
79 + dist.broadcast_object_list([seed], writer_rank)
80 + else:
81 + recv = [None]
82 + dist.broadcast_object_list(recv, writer_rank)
83 + seed = recv[0] # type: ignore
84 +
85 + if pg == dist.group.WORLD:
86 + dist.barrier()
87 + else:
88 + pg.barrier()
89 +
90 + # in case we find a race condition
91 + # print the seed so that we can reproduce the error
92 + print(f"Rank {rank} got seed {seed}")
93 + # test broadcasting with about 400MB of data
94 + N = 10_000
95 + if rank == writer_rank:
96 + arrs = get_arrays(N, seed)
97 + for x in arrs:
98 + broadcaster.broadcast_object(x)
99 + time.sleep(random.random() / 1000)
100 + else:
101 + arrs = get_arrays(N, seed)
102 + for x in arrs:
103 + y = broadcaster.broadcast_object(None)
104 + assert np.array_equal(x, y)
105 + time.sleep(random.random() / 1000)
106 +
107 + if pg == dist.group.WORLD:
108 + dist.barrier()
109 + print("torch distributed passed the test!")
110 + else:
111 + pg.barrier()
112 + print("StatelessProcessGroup passed the test!")
113
114
115 def test_shm_broadcast():
vllm/distributed/device_communicators/shm_broadcast.py
+26 -13
@@ -5,7 +5,7 @@ import time
5 from contextlib import contextmanager
6 from dataclasses import dataclass, field
7 from multiprocessing import shared_memory
8 -from typing import List, Optional, Tuple
8 +from typing import List, Optional, Tuple, Union
9 from unittest.mock import patch
10
11 import torch
@@ -15,6 +15,7 @@ from zmq import IPV6 # type: ignore
15 from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore
16
17 import vllm.envs as envs
18 +from vllm.distributed.utils import StatelessProcessGroup
19 from vllm.logger import init_logger
20 from vllm.utils import get_ip, get_open_port, is_valid_ipv6_address
21
@@ -476,13 +477,19 @@ class MessageQueue:
477 return self.dequeue()
478
479 @staticmethod
479 - def create_from_process_group(pg: ProcessGroup,
480 + def create_from_process_group(pg: Union[ProcessGroup,
481 + StatelessProcessGroup],
482 max_chunk_bytes,
483 max_chunks,
484 writer_rank=0) -> "MessageQueue":
483 - group_rank = dist.get_rank(pg)
484 - group_world_size = dist.get_world_size(pg)
485 - global_ranks = dist.get_process_group_ranks(pg)
485 + if isinstance(pg, ProcessGroup):
486 + group_rank = dist.get_rank(pg)
487 + group_world_size = dist.get_world_size(pg)
488 + global_ranks = dist.get_process_group_ranks(pg)
489 + else:
490 + group_rank = pg.rank
491 + group_world_size = pg.world_size
492 + global_ranks = list(range(pg.world_size))
493
494 from vllm.distributed.parallel_state import in_the_same_node_as
495 status = in_the_same_node_as(pg, source_rank=writer_rank)
@@ -500,15 +507,21 @@ class MessageQueue:
507 max_chunks=max_chunks,
508 )
509 handle = buffer_io.export_handle()
503 - dist.broadcast_object_list([handle],
504 - src=global_ranks[writer_rank],
505 - group=pg)
510 + if isinstance(pg, ProcessGroup):
511 + dist.broadcast_object_list([handle],
512 + src=global_ranks[writer_rank],
513 + group=pg)
514 + else:
515 + pg.broadcast_obj(handle, writer_rank)
516 else:
507 - recv = [None]
508 - dist.broadcast_object_list(recv,
509 - src=global_ranks[writer_rank],
510 - group=pg)
511 - handle = recv[0] # type: ignore
517 + if isinstance(pg, ProcessGroup):
518 + recv = [None]
519 + dist.broadcast_object_list(recv,
520 + src=global_ranks[writer_rank],
521 + group=pg)
522 + handle = recv[0] # type: ignore
523 + else:
524 + handle = pg.broadcast_obj(None, writer_rank)
525 buffer_io = MessageQueue.create_from_handle(handle, group_rank)
526 buffer_io.wait_until_ready()
527 return buffer_io
vllm/distributed/parallel_state.py
+43 -21
@@ -37,6 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37
38 import vllm.distributed.kv_transfer.kv_transfer_agent as kv_transfer
39 import vllm.envs as envs
40 +from vllm.distributed.utils import StatelessProcessGroup
41 from vllm.logger import init_logger
42 from vllm.platforms import current_platform
43 from vllm.utils import direct_register_custom_op, supports_custom_op
@@ -1191,25 +1192,31 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1192 torch.cuda.empty_cache()
1193
1194
1194 -def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1195 +def in_the_same_node_as(pg: Union[ProcessGroup, StatelessProcessGroup],
1196 + source_rank: int = 0) -> List[bool]:
1197 """
1198 This is a collective operation that returns if each rank is in the same node
1199 as the source rank. It tests if processes are attached to the same
1200 memory system (shared access to shared memory).
1201 """
1200 - assert torch.distributed.get_backend(
1201 - pg) != torch.distributed.Backend.NCCL, (
1202 - "in_the_same_node_as should be tested with a non-NCCL group.")
1203 - # local rank inside the group
1204 - rank = torch.distributed.get_rank(group=pg)
1205 - world_size = torch.distributed.get_world_size(group=pg)
1202 + if isinstance(pg, ProcessGroup):
1203 + assert torch.distributed.get_backend(
1204 + pg) != torch.distributed.Backend.NCCL, (
1205 + "in_the_same_node_as should be tested with a non-NCCL group.")
1206 + # local rank inside the group
1207 + rank = torch.distributed.get_rank(group=pg)
1208 + world_size = torch.distributed.get_world_size(group=pg)
1209 +
1210 + # global ranks of the processes in the group
1211 + ranks = torch.distributed.get_process_group_ranks(pg)
1212 + else:
1213 + rank = pg.rank
1214 + world_size = pg.world_size
1215 + ranks = list(range(world_size))
1216
1217 # local tensor in each process to store the result
1218 is_in_the_same_node = torch.tensor([0] * world_size, dtype=torch.int32)
1219
1210 - # global ranks of the processes in the group
1211 - ranks = torch.distributed.get_process_group_ranks(pg)
1212 -
1220 magic_message = b"magic_message"
1221 shm = None
1222
@@ -1219,17 +1226,21 @@ def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1226 # create a shared memory segment
1227 shm = shared_memory.SharedMemory(create=True, size=128)
1228 shm.buf[:len(magic_message)] = magic_message
1222 - torch.distributed.broadcast_object_list([shm.name],
1223 - src=ranks[source_rank],
1224 - group=pg)
1229 + if isinstance(pg, ProcessGroup):
1230 + torch.distributed.broadcast_object_list(
1231 + [shm.name], src=ranks[source_rank], group=pg)
1232 + else:
1233 + pg.broadcast_obj(shm.name, src=source_rank)
1234 is_in_the_same_node[rank] = 1
1235 else:
1236 # try to open the shared memory segment
1228 - recv = [None]
1229 - torch.distributed.broadcast_object_list(recv,
1230 - src=ranks[source_rank],
1231 - group=pg)
1232 - name = recv[0]
1237 + if isinstance(pg, ProcessGroup):
1238 + recv = [None]
1239 + torch.distributed.broadcast_object_list(
1240 + recv, src=ranks[source_rank], group=pg)
1241 + name = recv[0]
1242 + else:
1243 + name = pg.broadcast_obj(None, src=source_rank)
1244 # fix to https://stackoverflow.com/q/62748654/9191338
1245 # Python incorrectly tracks shared memory even if it is not
1246 # created by the process. The following patch is a workaround.
@@ -1244,12 +1255,23 @@ def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1255 if shm:
1256 shm.close()
1257
1247 - torch.distributed.barrier(group=pg)
1258 + if isinstance(pg, ProcessGroup):
1259 + torch.distributed.barrier(group=pg)
1260 + else:
1261 + pg.barrier()
1262
1263 # clean up the shared memory segment
1264 with contextlib.suppress(OSError):
1265 if rank == source_rank and shm:
1266 shm.unlink()
1253 - torch.distributed.all_reduce(is_in_the_same_node, group=pg)
1267
1255 - return [x == 1 for x in is_in_the_same_node.tolist()]
1268 + if isinstance(pg, ProcessGroup):
1269 + torch.distributed.all_reduce(is_in_the_same_node, group=pg)
1270 + aggregated_data = is_in_the_same_node
1271 + else:
1272 + aggregated_data = torch.zeros_like(is_in_the_same_node)
1273 + for i in range(world_size):
1274 + rank_data = pg.broadcast_obj(is_in_the_same_node, src=i)
1275 + aggregated_data += rank_data
1276 +
1277 + return [x == 1 for x in aggregated_data.tolist()]