A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Distributed] enable multiple tp group (#4512)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
youkaichao committed
May 1, 2024 at 21:28 UTC
2a85f9300733c09ec90819bc6df4bff8f103fd67
4 files changed
+43
-4
.buildkite/test-pipeline.yaml
+8
-3
@@ -25,19 +25,24 @@ steps:
25
- label: Distributed Comm Ops Test
26
command: pytest -v -s test_comm_ops.py
27
working_dir: "/vllm-workspace/tests/distributed"
28
- num_gpus: 2 # only support 1 or 2 for now.
28
+ num_gpus: 2
29
30
- label: Distributed Tests
31
working_dir: "/vllm-workspace/tests/distributed"
32
- num_gpus: 2 # only support 1 or 2 for now.
32
+ num_gpus: 2
33
commands:
34
- - pytest -v -s test_pynccl.py
34
- pytest -v -s test_pynccl_library.py
35
- TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
36
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_basic_distributed_correctness.py
37
- TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_chunked_prefill_distributed.py
38
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_chunked_prefill_distributed.py
39
40
+- label: Distributed Tests (Multiple Groups)
41
+ working_dir: "/vllm-workspace/tests/distributed"
42
+ num_gpus: 4
43
+ commands:
44
+ - pytest -v -s test_pynccl.py
45
+
46
- label: Engine Test
47
command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
48
.buildkite/test-template.j2
+3
@@ -45,6 +45,9 @@ steps:
45
plugins:
46
- kubernetes:
47
podSpec:
48
+ {% if step.num_gpus %}
49
+ priorityClassName: gpu-priority-cls-{{ step.num_gpus }}
50
+ {% endif %}
51
volumes:
52
- name: dshm
53
emptyDir:
tests/distributed/test_pynccl.py
+28
@@ -58,6 +58,34 @@ def test_pynccl():
58
distributed_run(worker_fn, 2)
59
60
61
+@worker_fn_wrapper
62
+def multiple_tp_worker_fn():
63
+ device = torch.device(f"cuda:{torch.distributed.get_rank()}")
64
+ groups = [
65
+ torch.distributed.new_group(ranks=[0, 1], backend="gloo"),
66
+ torch.distributed.new_group(ranks=[2, 3], backend="gloo")
67
+ ]
68
+ group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
69
+ comm = NCCLCommunicator(group=group, device=device)
70
+ tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
71
+ # two groups can communicate independently
72
+ if torch.distributed.get_rank() in [0, 1]:
73
+ comm.all_reduce(tensor)
74
+ comm.all_reduce(tensor)
75
+ result = tensor.mean().cpu().item()
76
+ assert result == 4
77
+ else:
78
+ comm.all_reduce(tensor)
79
+ result = tensor.mean().cpu().item()
80
+ assert result == 2
81
+
82
+
83
+@pytest.mark.skipif(torch.cuda.device_count() < 4,
84
+ reason="Need at least 2 GPUs to run the test.")
85
+def test_pynccl_multiple_tp():
86
+ distributed_run(worker_fn, 4)
87
+
88
+
89
@worker_fn_wrapper
90
def worker_fn_with_cudagraph():
91
with torch.no_grad():
vllm/distributed/device_communicators/pynccl.py
+4
-1
@@ -232,6 +232,7 @@ class NCCLCommunicator:
232
assert dist.get_backend(group) != dist.Backend.NCCL, (
233
"NCCLCommunicator should be attached to a non-NCCL group.")
234
self.group = group
235
+ # note: this rank is the rank in the group
236
self.rank = dist.get_rank(group)
237
self.world_size = dist.get_world_size(group)
238
if self.rank == 0:
@@ -239,7 +240,9 @@ class NCCLCommunicator:
240
else:
241
self.unique_id = NcclUniqueId()
242
tensor = torch.ByteTensor(list(self.unique_id.internal))
242
- dist.broadcast(tensor, src=0, group=group)
243
+ ranks = dist.get_process_group_ranks(group)
244
+ # arg `src` in `broadcast` is the global rank
245
+ dist.broadcast(tensor, src=ranks[0], group=group)
246
byte_list = tensor.tolist()
247
for i, byte in enumerate(byte_list):
248
self.unique_id.internal[i] = byte