@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2a85f93007
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] enable multiple tp group (#4512)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

youkaichao committed May 1, 2024 at 21:28 UTC 2a85f9300733c09ec90819bc6df4bff8f103fd67
4 files changed +43 -4
.buildkite/test-pipeline.yaml
+8 -3
@@ -25,19 +25,24 @@ steps:
25 - label: Distributed Comm Ops Test
26 command: pytest -v -s test_comm_ops.py
27 working_dir: "/vllm-workspace/tests/distributed"
28 - num_gpus: 2 # only support 1 or 2 for now.
28 + num_gpus: 2
29
30 - label: Distributed Tests
31 working_dir: "/vllm-workspace/tests/distributed"
32 - num_gpus: 2 # only support 1 or 2 for now.
32 + num_gpus: 2
33 commands:
34 - - pytest -v -s test_pynccl.py
34 - pytest -v -s test_pynccl_library.py
35 - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
36 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_basic_distributed_correctness.py
37 - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_chunked_prefill_distributed.py
38 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_chunked_prefill_distributed.py
39
40 +- label: Distributed Tests (Multiple Groups)
41 + working_dir: "/vllm-workspace/tests/distributed"
42 + num_gpus: 4
43 + commands:
44 + - pytest -v -s test_pynccl.py
45 +
46 - label: Engine Test
47 command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
48
.buildkite/test-template.j2
+3
@@ -45,6 +45,9 @@ steps:
45 plugins:
46 - kubernetes:
47 podSpec:
48 + {% if step.num_gpus %}
49 + priorityClassName: gpu-priority-cls-{{ step.num_gpus }}
50 + {% endif %}
51 volumes:
52 - name: dshm
53 emptyDir:
tests/distributed/test_pynccl.py
+28
@@ -58,6 +58,34 @@ def test_pynccl():
58 distributed_run(worker_fn, 2)
59
60
61 +@worker_fn_wrapper
62 +def multiple_tp_worker_fn():
63 + device = torch.device(f"cuda:{torch.distributed.get_rank()}")
64 + groups = [
65 + torch.distributed.new_group(ranks=[0, 1], backend="gloo"),
66 + torch.distributed.new_group(ranks=[2, 3], backend="gloo")
67 + ]
68 + group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
69 + comm = NCCLCommunicator(group=group, device=device)
70 + tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
71 + # two groups can communicate independently
72 + if torch.distributed.get_rank() in [0, 1]:
73 + comm.all_reduce(tensor)
74 + comm.all_reduce(tensor)
75 + result = tensor.mean().cpu().item()
76 + assert result == 4
77 + else:
78 + comm.all_reduce(tensor)
79 + result = tensor.mean().cpu().item()
80 + assert result == 2
81 +
82 +
83 +@pytest.mark.skipif(torch.cuda.device_count() < 4,
84 + reason="Need at least 2 GPUs to run the test.")
85 +def test_pynccl_multiple_tp():
86 + distributed_run(worker_fn, 4)
87 +
88 +
89 @worker_fn_wrapper
90 def worker_fn_with_cudagraph():
91 with torch.no_grad():
vllm/distributed/device_communicators/pynccl.py
+4 -1
@@ -232,6 +232,7 @@ class NCCLCommunicator:
232 assert dist.get_backend(group) != dist.Backend.NCCL, (
233 "NCCLCommunicator should be attached to a non-NCCL group.")
234 self.group = group
235 + # note: this rank is the rank in the group
236 self.rank = dist.get_rank(group)
237 self.world_size = dist.get_world_size(group)
238 if self.rank == 0:
@@ -239,7 +240,9 @@ class NCCLCommunicator:
240 else:
241 self.unique_id = NcclUniqueId()
242 tensor = torch.ByteTensor(list(self.unique_id.internal))
242 - dist.broadcast(tensor, src=0, group=group)
243 + ranks = dist.get_process_group_ranks(group)
244 + # arg `src` in `broadcast` is the global rank
245 + dist.broadcast(tensor, src=ranks[0], group=group)
246 byte_list = tensor.tolist()
247 for i, byte in enumerate(byte_list):
248 self.unique_id.internal[i] = byte