@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 719c1ca468
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[core][distributed] add stateless_init_process_group (#10072)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 6, 2024 at 16:42 UTC 719c1ca468537d2be2616ddc3163236af7f5bd62
3 files changed +147 -3
.buildkite/test-pipeline.yaml
+1 -1
@@ -120,6 +120,7 @@ steps:
120 - tests/spec_decode/e2e/test_integration_dist_tp4
121 - tests/compile
122 commands:
123 + - pytest -v -s distributed/test_utils.py
124 - pytest -v -s compile/test_basic_correctness.py
125 - pytest -v -s distributed/test_pynccl.py
126 - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
@@ -431,7 +432,6 @@ steps:
432 - pip install -e ./plugins/vllm_add_dummy_model
433 - pytest -v -s distributed/test_distributed_oot.py
434 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
434 - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
435
436 - label: Multi-step Tests (4 GPUs) # 36min
437 working_dir: "/vllm-workspace/tests"
tests/distributed/test_utils.py
+73 -2
@@ -1,9 +1,15 @@
1 +import pytest
2 import ray
3 +import torch
4 +import torch.distributed as dist
5
6 import vllm.envs as envs
7 +from vllm.distributed.utils import stateless_init_process_group
8 from vllm.utils import (cuda_device_count_stateless,
9 update_environment_variables)
10
11 +from ..utils import multi_gpu_test
12 +
13
14 @ray.remote
15 class _CUDADeviceCountStatelessTestActor:
@@ -24,10 +30,75 @@ def test_cuda_device_count_stateless():
30 CUDA_VISIBLE_DEVICES is changed."""
31 actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
32 num_gpus=2).remote()
27 - assert sorted(ray.get(
28 - actor.get_cuda_visible_devices.remote()).split(",")) == ["0", "1"]
33 + assert len(
34 + sorted(ray.get(
35 + actor.get_cuda_visible_devices.remote()).split(","))) == 2
36 assert ray.get(actor.get_count.remote()) == 2
37 ray.get(actor.set_cuda_visible_devices.remote("0"))
38 assert ray.get(actor.get_count.remote()) == 1
39 ray.get(actor.set_cuda_visible_devices.remote(""))
40 assert ray.get(actor.get_count.remote()) == 0
41 +
42 +
43 +def cpu_worker(rank, WORLD_SIZE):
44 + pg1 = stateless_init_process_group(init_method="tcp://127.0.0.1:29500",
45 + rank=rank,
46 + world_size=WORLD_SIZE,
47 + backend="gloo")
48 + if rank <= 2:
49 + pg2 = stateless_init_process_group(init_method="tcp://127.0.0.1:29501",
50 + rank=rank,
51 + world_size=3,
52 + backend="gloo")
53 + data = torch.tensor([rank])
54 + dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg1)
55 + if rank <= 2:
56 + dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg2)
57 + item = data[0].item()
58 + print(f"rank: {rank}, item: {item}")
59 + if rank == 3:
60 + assert item == 6
61 + else:
62 + assert item == 18
63 +
64 +
65 +def gpu_worker(rank, WORLD_SIZE):
66 + pg1 = stateless_init_process_group(init_method="tcp://127.0.0.1:29502",
67 + rank=rank,
68 + world_size=WORLD_SIZE,
69 + backend="nccl")
70 + if rank <= 2:
71 + pg2 = stateless_init_process_group(init_method="tcp://127.0.0.1:29503",
72 + rank=rank,
73 + world_size=3,
74 + backend="nccl")
75 + torch.cuda.set_device(rank)
76 + data = torch.tensor([rank]).cuda()
77 + dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg1)
78 + if rank <= 2:
79 + dist.all_reduce(data, op=dist.ReduceOp.SUM, group=pg2)
80 + item = data[0].item()
81 + print(f"rank: {rank}, item: {item}")
82 + if rank == 3:
83 + assert item == 6
84 + else:
85 + assert item == 18
86 +
87 +
88 +@multi_gpu_test(num_gpus=4)
89 +@pytest.mark.parametrize("worker", [cpu_worker, gpu_worker])
90 +def test_stateless_init_process_group(worker):
91 + WORLD_SIZE = 4
92 + from multiprocessing import get_context
93 + ctx = get_context("fork")
94 + processes = []
95 + for i in range(WORLD_SIZE):
96 + rank = i
97 + processes.append(ctx.Process(target=worker, args=(rank, WORLD_SIZE)))
98 + for p in processes:
99 + p.start()
100 + for p in processes:
101 + p.join()
102 + for p in processes:
103 + assert not p.exitcode
104 + print("All processes finished.")
vllm/distributed/utils.py
+73
@@ -5,6 +5,11 @@
5 from typing import Sequence, Tuple
6
7 import torch
8 +from torch.distributed import ProcessGroup
9 +from torch.distributed.distributed_c10d import (Backend, PrefixStore,
10 + _get_default_timeout,
11 + is_nccl_available)
12 +from torch.distributed.rendezvous import rendezvous
13
14 import vllm.envs as envs
15 from vllm.logger import init_logger
@@ -84,3 +89,71 @@ def get_pp_indices(num_hidden_layers: int, pp_rank: int,
89 end_layer = num_hidden_layers
90
91 return (start_layer, end_layer)
92 +
93 +
94 +def stateless_init_process_group(init_method: str, rank: int, world_size: int,
95 + backend: str) -> ProcessGroup:
96 + """A replacement for `torch.distributed.init_process_group` that does not
97 + pollute the global state.
98 +
99 + If we have process A and process B called `torch.distributed.init_process_group`
100 + to form a group, and then we want to form another group with process A, B, C,
101 + D, it is not possible in PyTorch, because process A and process B have already
102 + formed a group, and process C and process D cannot join that group. This
103 + function is a workaround for this issue.
104 +
105 + `torch.distributed.init_process_group` is a global call, while this function
106 + is a stateless call. It will return a `ProcessGroup` object that can be used
107 + for collective communication. With this function, process A and process B
108 + can call `stateless_init_process_group` to form a group, and then process A, B,
109 + C, and D can call `stateless_init_process_group` to form another group.
110 + """ # noqa
111 +
112 + backend = Backend(backend) # it is basically string
113 + timeout = _get_default_timeout(backend)
114 +
115 + store, rank, world_size = next(
116 + rendezvous(init_method, rank, world_size, timeout=timeout))
117 + store.set_timeout(timeout)
118 +
119 + group_rank = rank
120 + group_size = world_size
121 +
122 + # Use a PrefixStore to avoid accidental overrides of keys used by
123 + # different systems (e.g. RPC) in case the store is multi-tenant.
124 + prefix_store = PrefixStore(init_method, store)
125 +
126 + pg_options = ProcessGroup.Options(backend=backend, timeout=timeout)
127 +
128 + pg: ProcessGroup = ProcessGroup(
129 + prefix_store,
130 + group_rank,
131 + group_size,
132 + pg_options,
133 + )
134 +
135 + if backend == "gloo":
136 + from torch.distributed.distributed_c10d import ProcessGroupGloo
137 + backend_class = ProcessGroupGloo(prefix_store,
138 + group_rank,
139 + group_size,
140 + timeout=timeout)
141 + backend_type = ProcessGroup.BackendType.GLOO
142 + device = torch.device("cpu")
143 + elif backend == "nccl":
144 + assert is_nccl_available()
145 + from torch.distributed.distributed_c10d import ProcessGroupNCCL
146 +
147 + backend_options = ProcessGroupNCCL.Options()
148 + backend_options._timeout = timeout
149 +
150 + backend_class = ProcessGroupNCCL(prefix_store, group_rank, group_size,
151 + backend_options)
152 + backend_type = ProcessGroup.BackendType.NCCL
153 + device = torch.device("cuda")
154 +
155 + backend_class._set_sequence_number_for_group()
156 +
157 + pg._register_backend(device, backend_type, backend_class)
158 +
159 + return pg