@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 702bee461f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] refactor custom allreduce to support multiple tp groups (#4754)

youkaichao committed May 12, 2024 at 17:47 UTC 702bee461f448b0186eb9d673baad29fd923c884
10 files changed +327 -226
tests/distributed/test_comm_ops.py
+11 -11
@@ -16,7 +16,7 @@ from vllm.test_utils import (init_test_distributed_environment,
16
17
18 @ray.remote(num_gpus=1, max_calls=1)
19 -def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
19 +def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
20 distributed_init_port: str):
21 # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
22 # so that each worker can see all the GPUs
@@ -24,12 +24,12 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
24 del os.environ["CUDA_VISIBLE_DEVICES"]
25 device = torch.device(f"cuda:{rank}")
26 torch.cuda.set_device(device)
27 - init_test_distributed_environment(1, tensor_parallel_size, rank,
27 + init_test_distributed_environment(tp_size, pp_size, rank,
28 distributed_init_port)
29 num_elements = 8
30 all_tensors = [
31 torch.arange(num_elements, dtype=torch.float32, device="cuda") *
32 - (r + 1) for r in range(tensor_parallel_size)
32 + (r + 1) for r in range(tp_size)
33 ]
34 expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
35 t = all_tensors[rank]
@@ -38,7 +38,7 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
38
39
40 @ray.remote(num_gpus=1, max_calls=1)
41 -def all_gather_test_worker(tensor_parallel_size: int, rank: int,
41 +def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
42 distributed_init_port: str):
43 # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
44 # so that each worker can see all the GPUs
@@ -46,7 +46,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
46 del os.environ["CUDA_VISIBLE_DEVICES"]
47 device = torch.device(f"cuda:{rank}")
48 torch.cuda.set_device(device)
49 - init_test_distributed_environment(1, tensor_parallel_size, rank,
49 + init_test_distributed_environment(tp_size, pp_size, rank,
50 distributed_init_port)
51 num_dimensions = 3
52 tensor_size = list(range(2, num_dimensions + 2))
@@ -57,7 +57,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
57 all_tensors = [
58 torch.arange(total_size, dtype=torch.float32,
59 device="cuda").reshape(tensor_size) * (r + 1)
60 - for r in range(tensor_parallel_size)
60 + for r in range(tp_size)
61 ]
62 expected = torch.cat(all_tensors, dim=all_gather_dimension)
63 t = all_tensors[rank]
@@ -66,7 +66,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
66
67
68 @ray.remote(num_gpus=1, max_calls=1)
69 -def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
69 +def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
70 distributed_init_port: str):
71 # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
72 # so that each worker can see all the GPUs
@@ -74,7 +74,7 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
74 del os.environ["CUDA_VISIBLE_DEVICES"]
75 device = torch.device(f"cuda:{rank}")
76 torch.cuda.set_device(device)
77 - init_test_distributed_environment(1, tensor_parallel_size, rank,
77 + init_test_distributed_environment(tp_size, pp_size, rank,
78 distributed_init_port)
79 test_dict = {
80 # device tensor
@@ -106,10 +106,10 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
106
107 @pytest.mark.skipif(torch.cuda.device_count() < 2,
108 reason="Need at least 2 GPUs to run the test.")
109 -@pytest.mark.parametrize("tensor_parallel_size", [2])
109 +@pytest.mark.parametrize("tp_size", [2])
110 @pytest.mark.parametrize("test_target", [
111 all_reduce_test_worker, all_gather_test_worker,
112 broadcast_tensor_dict_test_worker
113 ])
114 -def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
115 - multi_process_tensor_parallel(tensor_parallel_size, test_target)
114 +def test_multi_process_tensor_parallel(tp_size, test_target):
115 + multi_process_tensor_parallel(tp_size, 1, test_target)
tests/distributed/test_custom_all_reduce.py
+58 -29
@@ -6,8 +6,10 @@ import ray
6 import torch
7 import torch.distributed as dist
8
9 -from vllm.distributed import tensor_model_parallel_all_reduce
10 -from vllm.distributed.device_communicators import custom_all_reduce
9 +from vllm.distributed.communication_op import ( # noqa
10 + graph_capture, tensor_model_parallel_all_reduce)
11 +from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12 + get_tp_ca_communicator)
13 from vllm.test_utils import (init_test_distributed_environment,
14 multi_process_tensor_parallel)
15
@@ -18,17 +20,36 @@ for i, v in enumerate(test_sizes):
20
21
22 @ray.remote(num_gpus=1, max_calls=1)
21 -def graph_allreduce(world_size, rank, distributed_init_port):
23 +def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
24 del os.environ["CUDA_VISIBLE_DEVICES"]
25 device = torch.device(f"cuda:{rank}")
26 torch.cuda.set_device(device)
25 - init_test_distributed_environment(1, world_size, rank,
27 + init_test_distributed_environment(tp_size, pp_size, rank,
28 distributed_init_port)
29
28 - custom_all_reduce.init_custom_ar()
30 + group = get_tensor_model_parallel_group()
31 +
32 + # A small all_reduce for warmup.
33 + # this is needed because device communicators might be created lazily
34 + # (e.g. NCCL). This will ensure that the communicator is initialized
35 + # before any communication happens, so that this group can be used for
36 + # graph capture immediately.
37 + data = torch.zeros(1)
38 + data = data.to(device=device)
39 + torch.distributed.all_reduce(data, group=group)
40 + torch.cuda.synchronize()
41 + del data
42 +
43 + # we use the first group to communicate once
44 + # and the second group to communicate twice
45 + # and so on
46 + # this is used to demonstrate that each group can
47 + # communicate independently
48 + num_communication = rank // tp_size + 1
49 +
50 for sz in test_sizes:
51 for dtype in [torch.float32, torch.float16, torch.bfloat16]:
31 - with custom_all_reduce.capture():
52 + with graph_capture():
53 # use integers so result matches NCCL exactly
54 inp1 = torch.randint(1,
55 16, (sz, ),
@@ -41,44 +62,52 @@ def graph_allreduce(world_size, rank, distributed_init_port):
62 torch.cuda.synchronize()
63 graph = torch.cuda.CUDAGraph()
64 with torch.cuda.graph(graph):
44 - out1 = tensor_model_parallel_all_reduce(inp1)
45 - # the input buffer is immediately modified to test
46 - # synchronization
47 - dist.all_reduce(inp1)
48 - out2 = tensor_model_parallel_all_reduce(inp2)
49 - dist.all_reduce(inp2)
65 + for i in range(num_communication):
66 + out1 = tensor_model_parallel_all_reduce(inp1)
67 + # the input buffer is immediately modified to test
68 + # synchronization
69 + dist.all_reduce(inp1, group=group)
70 + out2 = tensor_model_parallel_all_reduce(inp2)
71 + dist.all_reduce(inp2, group=group)
72 graph.replay()
73 assert torch.allclose(out1, inp1)
74 assert torch.allclose(out2, inp2)
75
76
77 @ray.remote(num_gpus=1, max_calls=1)
56 -def eager_allreduce(world_size, rank, distributed_init_port):
78 +def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
79 del os.environ["CUDA_VISIBLE_DEVICES"]
80 device = torch.device(f"cuda:{rank}")
81 torch.cuda.set_device(device)
60 - init_test_distributed_environment(1, world_size, rank,
82 + init_test_distributed_environment(tp_size, pp_size, rank,
83 distributed_init_port)
84
85 + # we use the first group to communicate once
86 + # and the second group to communicate twice
87 + # and so on
88 + # this is used to demonstrate that each group can
89 + # communicate independently
90 + num_communication = rank // tp_size + 1
91 sz = 1024
64 - custom_all_reduce.init_custom_ar()
65 - fa = custom_all_reduce.get_handle()
92 + fa = get_tp_ca_communicator()
93 inp = torch.ones(sz, dtype=torch.float32, device=device)
67 - out = fa.all_reduce_unreg(inp)
68 - assert torch.allclose(out, inp * world_size)
94 + out = inp
95 + for _ in range(num_communication):
96 + out = fa.all_reduce_unreg(out)
97 + assert torch.allclose(out, inp * (tp_size**num_communication))
98
99 inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
71 - out = fa.all_reduce_unreg(inp)
72 - assert torch.allclose(out, inp * world_size)
100 + out = inp
101 + for _ in range(num_communication):
102 + out = fa.all_reduce_unreg(out)
103 + assert torch.allclose(out, inp * (tp_size**num_communication))
104
105
75 -@pytest.mark.skipif(torch.cuda.device_count() < 2,
76 - reason="Need at least 2 GPUs to run the test.")
77 -@pytest.mark.parametrize("tensor_parallel_size", [2])
106 +@pytest.mark.parametrize("tp_size", [2])
107 +@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
108 @pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
79 -def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
80 - multi_process_tensor_parallel(tensor_parallel_size, test_target)
81 -
82 -
83 -if __name__ == "__main__":
84 - multi_process_tensor_parallel(2, graph_allreduce)
109 +def test_custom_allreduce(tp_size, pipeline_parallel_size, test_target):
110 + world_size = tp_size * pipeline_parallel_size
111 + if world_size > torch.cuda.device_count():
112 + pytest.skip("Not enough GPUs to run the test.")
113 + multi_process_tensor_parallel(tp_size, pipeline_parallel_size, test_target)
tests/distributed/test_pynccl.py
+2 -2
@@ -5,7 +5,7 @@ import pytest
5 import torch
6
7 from vllm.distributed.communication_op import ( # noqa
8 - graph_capture_mode, tensor_model_parallel_all_reduce)
8 + graph_mode, tensor_model_parallel_all_reduce)
9 from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
10 from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
11 from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
@@ -103,7 +103,7 @@ def multiple_tp_with_vllm_worker_fn():
103 device = torch.device(f"cuda:{torch.distributed.get_rank()}")
104 ensure_model_parallel_initialized(2, 2)
105 tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
106 - with graph_capture_mode():
106 + with graph_mode():
107 # two tp groups can communicate independently
108 if torch.distributed.get_rank() in [0, 1]:
109 tensor = tensor_model_parallel_all_reduce(tensor)
vllm/distributed/communication_op.py
+34 -11
@@ -1,5 +1,5 @@
1 from collections import namedtuple
2 -from contextlib import contextmanager
2 +from contextlib import contextmanager, nullcontext
3 from typing import Any, Dict, List, Optional, Tuple, Union
4
5 import torch
@@ -9,12 +9,13 @@ from .parallel_state import (get_cpu_world_group,
9 get_tensor_model_parallel_group,
10 get_tensor_model_parallel_rank,
11 get_tensor_model_parallel_world_size,
12 + get_tp_ca_communicator,
13 get_tp_pynccl_communicator)
14
15
16 @contextmanager
16 -def graph_capture_mode():
17 - # In graph capture, we have to be very careful about the collective
17 +def graph_mode():
18 + # In graph mode, we have to be very careful about the collective
19 # operations. The current status is:
20 # allreduce \ Mode | Eager | Graph |
21 # --------------------------------------------
@@ -24,10 +25,32 @@ def graph_capture_mode():
25 #
26 # Note that custom allreduce will have a runtime check, if the tensor size
27 # is too large, it will fallback to the next available option.
28 + # In summary: When using CUDA graph, we use
29 + # either custom all-reduce kernel or pynccl. When not using CUDA
30 + # graph, we use either custom all-reduce kernel or PyTorch NCCL.
31 + # We always prioritize using custom all-reduce kernel but fall back
32 + # to PyTorch or pynccl if it is disabled or not supported.
33 pynccl_comm = get_tp_pynccl_communicator()
28 - assert pynccl_comm is not None
29 - with pynccl_comm.change_state(enable=True,
30 - stream=torch.cuda.current_stream()):
34 + if pynccl_comm is None:
35 + context = nullcontext()
36 + else:
37 + context = pynccl_comm.change_state(enable=True,
38 + stream=torch.cuda.current_stream())
39 + with context:
40 + yield
41 +
42 +
43 +@contextmanager
44 +def graph_capture():
45 + """
46 + `graph_capture` is a context manager which should include the code that
47 + is capturing the CUDA graph. Its main purpose is to ensure that the
48 + some operations will be run after the graph is captured, before the graph
49 + is replayed.
50 + """
51 + ca_comm = get_tp_ca_communicator()
52 + context = nullcontext() if ca_comm is None else ca_comm.capture()
53 + with context:
54 yield
55
56
@@ -43,15 +66,15 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
66 TLDR: always assume this function modifies its input, but use the return
67 value as the output.
68 """
46 - from vllm.distributed.device_communicators.custom_all_reduce import (
47 - custom_all_reduce)
69 + ca_comm = get_tp_ca_communicator()
70
71 # Bypass the function if we are using only 1 GPU.
72 if get_tensor_model_parallel_world_size() == 1:
73 return input_
52 - out = custom_all_reduce(input_)
53 - if out is not None:
54 - return out
74 + if ca_comm is not None:
75 + out = ca_comm.custom_all_reduce(input_)
76 + if out is not None:
77 + return out
78 pynccl_comm = get_tp_pynccl_communicator()
79 if (pynccl_comm is not None and not pynccl_comm.disabled):
80 pynccl_comm.all_reduce(input_)
vllm/distributed/device_communicators/custom_all_reduce.py
+176 -144
@@ -1,154 +1,42 @@
1 from contextlib import contextmanager
2 -from typing import Any, List, Optional
2 +from typing import Any, List, Optional, Union
3
4 import torch
5 import torch.distributed as dist
6 +from torch.distributed import ProcessGroup
7
8 import vllm.envs as envs
9 +from vllm.distributed.parallel_state import (
10 + get_local_rank, get_tensor_model_parallel_cpu_group)
11 from vllm.logger import init_logger
12
13 try:
14 import pynvml
15
16 from vllm._C import custom_ar
17 +
18 + @contextmanager
19 + def _nvml():
20 + try:
21 + pynvml.nvmlInit()
22 + yield
23 + finally:
24 + pynvml.nvmlShutdown()
25 +
26 except ImportError:
27 # For AMD GPUs
28 custom_ar = None
29 pynvml = None
30
19 -logger = init_logger(__name__)
31 + @contextmanager
32 + def _nvml():
33 + try:
34 + yield
35 + finally:
36 + pass
37
21 -_CA_HANDLE: Optional["CustomAllreduce"] = None
22 -_IS_CAPTURING = False
23 -_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
24 -
25 -
26 -def init_custom_ar() -> None:
27 - from vllm.distributed import (get_tensor_model_parallel_rank,
28 - get_tensor_model_parallel_world_size)
29 -
30 - global _CA_HANDLE
31 - if _CA_HANDLE is not None:
32 - return
33 - rank = get_tensor_model_parallel_rank()
34 - world_size = get_tensor_model_parallel_world_size()
35 - if world_size == 1:
36 - # No need to initialize custom allreduce for single GPU case.
37 - return
38 -
39 - if world_size not in _SUPPORTED_WORLD_SIZES:
40 - logger.warning(
41 - "Custom allreduce is disabled due to an unsupported world size: "
42 - "%d. Supported world sizes: %s. To silence this warning, specify"
43 - " disable_custom_all_reduce=True explicitly.", world_size,
44 - str(_SUPPORTED_WORLD_SIZES))
45 - return
46 - num_dev = torch.cuda.device_count()
47 - # note: num dev can be larger than world_size if we're only using
48 - # first few GPUs
49 - if num_dev < world_size:
50 - logger.warning(
51 - "Cannot test GPU P2P because not all GPUs are visible to the "
52 - "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
53 - " is set.")
54 - return
55 -
56 - # we only use a subset of GPUs here
57 - # so we only need to check the nvlink connectivity of these GPUs
58 - num_dev = world_size
59 - # test nvlink first, this will filter out most of the cases
60 - # where custom allreduce is not supported
61 - cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
62 - if cuda_visible_devices:
63 - device_ids = list(map(int, cuda_visible_devices.split(",")))
64 - else:
65 - device_ids = list(range(num_dev))
66 - # this checks hardware and driver support for NVLink
67 - full_nvlink = _is_full_nvlink(device_ids)
68 - if world_size > 2 and not full_nvlink:
69 - logger.warning(
70 - "Custom allreduce is disabled because it's not supported on more"
71 - " than two PCIe-only GPUs. To silence this warning, specify"
72 - " disable_custom_all_reduce=True explicitly.")
73 - return
74 - # test P2P capability, this checks software/cudaruntime support
75 - # this is expensive to compute at the first time
76 - # then we cache the result
77 - if not _can_p2p(rank, world_size):
78 - logger.warning(
79 - "Custom allreduce is disabled because your platform lacks GPU P2P"
80 - " capability or P2P test failed. To silence this warning, specify"
81 - " disable_custom_all_reduce=True explicitly.")
82 - return
83 - _CA_HANDLE = CustomAllreduce(rank, world_size, full_nvlink)
84 -
85 -
86 -def begin_capture() -> None:
87 - global _IS_CAPTURING
88 - _IS_CAPTURING = True
89 -
90 -
91 -def end_capture() -> None:
92 - global _IS_CAPTURING
93 - _IS_CAPTURING = False
94 -
95 -
96 -def is_capturing() -> bool:
97 - return _IS_CAPTURING and _CA_HANDLE is not None
98 -
99 -
100 -def get_handle() -> Optional["CustomAllreduce"]:
101 - return _CA_HANDLE
102 -
103 -
104 -def is_initialized() -> bool:
105 - return _CA_HANDLE is not None
106 -
107 -
108 -@contextmanager
109 -def capture():
110 - try:
111 - begin_capture()
112 - yield
113 - finally:
114 - end_capture()
115 - handle = get_handle()
116 - if handle is not None:
117 - handle.register_graph_buffers()
118 -
119 -
120 -def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
121 - ca_handle = get_handle()
122 - # when custom allreduce is disabled, this will be None
123 - if ca_handle is None:
124 - return None
125 - if is_capturing():
126 - if torch.cuda.is_current_stream_capturing():
127 - if ca_handle.should_custom_ar(input):
128 - return ca_handle.all_reduce_reg(input)
129 - else:
130 - if ca_handle.should_custom_ar(input):
131 - # if warm up, mimic the allocation pattern
132 - # since custom allreduce is out-of-place
133 - return torch.empty_like(input)
134 - else:
135 - # note: outside of cuda graph context,
136 - # custom allreduce incurs a cost of cudaMemcpy, which should
137 - # be small(<=1% of overall latency) compared to the performance
138 - # gains of using custom kernels
139 - if ca_handle.should_custom_ar(input):
140 - return ca_handle.all_reduce_unreg(input)
141 -
142 - return None
143 -
144 -
145 -@contextmanager
146 -def _nvml():
147 - try:
148 - pynvml.nvmlInit()
149 - yield
150 - finally:
151 - pynvml.nvmlShutdown()
38 +
39 +logger = init_logger(__name__)
40
41
42 @_nvml()
@@ -188,22 +76,112 @@ def _can_p2p(rank: int, world_size: int) -> bool:
76
77 class CustomAllreduce:
78
79 + _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
80 +
81 # max_size: max supported allreduce size
82 def __init__(self,
193 - rank,
194 - world_size,
195 - full_nvlink,
83 + group: Optional[ProcessGroup] = None,
84 + device: Optional[Union[int, str, torch.device]] = None,
85 max_size=8192 * 1024) -> None:
86 + """
87 + Args:
88 + group: the process group to work on. If None, it will use the
89 + default process group.
90 + device: the device to bind the CustomAllreduce to. If None,
91 + it will be bind to f"cuda:{local_rank}".
92 + It is the caller's responsibility to make sure each communicator
93 + is bind to a unique device, and all communicators in this group
94 + are in the same node.
95 + """
96 + self._IS_CAPTURING = False
97 + self.disabled = True
98 +
99 + if custom_ar is None:
100 + # disable because of missing custom allreduce library
101 + # e.g. in a non-cuda environment
102 + return
103 +
104 + group = group or get_tensor_model_parallel_cpu_group()
105 + self.group = group
106 +
107 + assert dist.get_backend(group) != dist.Backend.NCCL, (
108 + "CustomAllreduce should be attached to a non-NCCL group.")
109 +
110 + rank = dist.get_rank(group=self.group)
111 + world_size = dist.get_world_size(group=self.group)
112 + if world_size == 1:
113 + # No need to initialize custom allreduce for single GPU case.
114 + return
115 +
116 + if world_size not in CustomAllreduce._SUPPORTED_WORLD_SIZES:
117 + logger.warning(
118 + "Custom allreduce is disabled due to an unsupported world"
119 + " size: %d. Supported world sizes: %s. To silence this "
120 + "warning, specify disable_custom_all_reduce=True explicitly.",
121 + world_size, str(CustomAllreduce._SUPPORTED_WORLD_SIZES))
122 + return
123 +
124 + if device is None:
125 + local_rank = get_local_rank()
126 + device = torch.device(f"cuda:{local_rank}")
127 + elif isinstance(device, int):
128 + device = torch.device(f"cuda:{device}")
129 + elif isinstance(device, str):
130 + device = torch.device(device)
131 + # now `device` is a `torch.device` object
132 + assert isinstance(device, torch.device)
133 + self.device = device
134 +
135 + cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
136 + if cuda_visible_devices:
137 + device_ids = list(map(int, cuda_visible_devices.split(",")))
138 + else:
139 + device_ids = list(range(torch.cuda.device_count()))
140 +
141 + physical_device_id = device_ids[device.index]
142 + tensor = torch.tensor([physical_device_id],
143 + dtype=torch.int,
144 + device="cpu")
145 + gather_list = [
146 + torch.tensor([0], dtype=torch.int, device="cpu")
147 + for _ in range(world_size)
148 + ]
149 + dist.all_gather(gather_list, tensor, group=self.group)
150 + physical_device_ids = [t.item() for t in gather_list]
151 +
152 + # test nvlink first, this will filter out most of the cases
153 + # where custom allreduce is not supported
154 + # this checks hardware and driver support for NVLink
155 + full_nvlink = _is_full_nvlink(physical_device_ids)
156 + if world_size > 2 and not full_nvlink:
157 + logger.warning(
158 + "Custom allreduce is disabled because it's not supported on"
159 + " more than two PCIe-only GPUs. To silence this warning, "
160 + "specify disable_custom_all_reduce=True explicitly.")
161 + return
162 + # test P2P capability, this checks software/cudaruntime support
163 + # this is expensive to compute at the first time
164 + # then we cache the result
165 + if not _can_p2p(rank, world_size):
166 + logger.warning(
167 + "Custom allreduce is disabled because your platform lacks "
168 + "GPU P2P capability or P2P test failed. To silence this "
169 + "warning, specify disable_custom_all_reduce=True explicitly.")
170 + return
171 +
172 + self.disabled = False
173 # buffers memory are owned by this Python class and passed to C++
174 # meta data composes of two parts: meta data for synchronization
175 # (256 bytes) and a temporary buffer for storing intermediate
176 # allreduce results.
177 self.meta = torch.zeros(custom_ar.meta_size() + max_size,
178 dtype=torch.uint8,
203 - device="cuda")
179 + device=self.device)
180 # This is a pre-registered IPC buffer. In eager mode, input tensors
181 # are first copied into this buffer before allreduce is performed
206 - self.buffer = torch.empty(max_size, dtype=torch.uint8, device="cuda")
182 + self.buffer = torch.empty(max_size,
183 + dtype=torch.uint8,
184 + device=self.device)
185 # This is a buffer for storing the tuples of pointers pointing to
186 # IPC buffers from all ranks. Each registered tuple has size of
187 # 8*world_size bytes where world_size is at most 8. Allocating 8MB
@@ -211,8 +189,9 @@ class CustomAllreduce:
189 # needs less than 10000 of registered tuples.
190 self.rank_data = torch.empty(8 * 1024 * 1024,
191 dtype=torch.uint8,
214 - device="cuda")
192 + device=self.device)
193 self.max_size = max_size
194 + self.rank = rank
195 self.world_size = world_size
196 handles, offsets = self._get_ipc_meta(self.meta)
197 self.full_nvlink = full_nvlink
@@ -221,6 +200,21 @@ class CustomAllreduce:
200 self.full_nvlink)
201 self.register_buffer(self.buffer)
202
203 + @contextmanager
204 + def capture(self):
205 + """
206 + The main responsibility of this context manager is the
207 + `register_graph_buffers` call at the end of the context.
208 + It records all the buffer addresses used in the CUDA graph.
209 + """
210 + try:
211 + self._IS_CAPTURING = True
212 + yield
213 + finally:
214 + self._IS_CAPTURING = False
215 + if not self.disabled:
216 + self.register_graph_buffers()
217 +
218 def _get_ipc_meta(self, inp: torch.Tensor):
219 data = inp.untyped_storage()._share_cuda_()
220 shard_data = (
@@ -230,14 +224,29 @@ class CustomAllreduce:
224 return self._gather_ipc_meta(shard_data)
225
226 def _gather_ipc_meta(self, shard_data):
233 - all_data: List[Optional[Any]] = [None] * self.world_size
234 - dist.all_gather_object(all_data, shard_data)
227 + # Note: don't use `[[None]] * self.world_size` here
228 + # because it will create a list of the same reference
229 + all_data: List[Optional[Any]] = [[None]
230 + for i in range(self.world_size)]
231 + all_data[self.rank][0] = shard_data
232 +
233 + ranks = dist.get_process_group_ranks(group=self.group)
234 + ranks.sort()
235 + for i, rank in enumerate(ranks):
236 + dist.broadcast_object_list(all_data[i],
237 + src=rank,
238 + group=self.group,
239 + device="cpu")
240 +
241 + # we cannot directly use `dist.all_gather_object` here
242 + # because it is incompatible with `gloo` backend under inference mode.
243 + # see https://github.com/pytorch/pytorch/issues/126032 for details.
244
245 handles = []
246 offsets = []
247 for i in range(len(all_data)):
239 - handles.append(all_data[i][0]) # type: ignore
240 - offsets.append(all_data[i][1]) # type: ignore
248 + handles.append(all_data[i][0][0]) # type: ignore
249 + offsets.append(all_data[i][0][1]) # type: ignore
250 return handles, offsets
251
252 def register_buffer(self, inp: torch.Tensor):
@@ -269,8 +278,31 @@ class CustomAllreduce:
278 custom_ar.all_reduce_unreg(self._ptr, inp, self.buffer, out)
279 return out
280
281 + def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
282 + # when custom allreduce is disabled, this will be None
283 + if self.disabled:
284 + return None
285 + if self._IS_CAPTURING:
286 + if torch.cuda.is_current_stream_capturing():
287 + if self.should_custom_ar(input):
288 + return self.all_reduce_reg(input)
289 + else:
290 + if self.should_custom_ar(input):
291 + # if warm up, mimic the allocation pattern
292 + # since custom allreduce is out-of-place
293 + return torch.empty_like(input)
294 + else:
295 + # note: outside of cuda graph context,
296 + # custom allreduce incurs a cost of cudaMemcpy, which should
297 + # be small(<=1% of overall latency) compared to the performance
298 + # gains of using custom kernels
299 + if self.should_custom_ar(input):
300 + return self.all_reduce_unreg(input)
301 +
302 + return None
303 +
304 def close(self):
273 - if self._ptr:
305 + if not self.disabled and self._ptr:
306 custom_ar.dispose(self._ptr)
307 self._ptr = 0
308
vllm/distributed/device_communicators/pynccl.py
+3 -1
@@ -96,8 +96,10 @@ class PyNcclCommunicator:
96 self.stream = torch.cuda.Stream()
97
98 # A small all_reduce for warmup.
99 - self.all_reduce(torch.zeros(1, device=device))
99 + data = torch.zeros(1, device=device)
100 + self.all_reduce(data)
101 self.stream.synchronize()
102 + del data
103
104 # by default it is disabled, e.g. in profiling models and prefill phase.
105 # to use it, use under `with obj.change_state(enable=True)`, usually
vllm/distributed/parallel_state.py
+27 -1
@@ -13,10 +13,13 @@ from vllm.logger import init_logger
13
14 logger = init_logger(__name__)
15
16 +_ENABLE_CUSTOM_ALL_REDUCE = True
17 +
18 # Tensor model parallel group that the current rank belongs to.
19 _TP_DEVICE_GROUP: Optional[ProcessGroup] = None
20 _TP_CPU_GROUP: Optional[ProcessGroup] = None
21 _TP_PYNCCL_COMMUNICATOR = None
22 +_TP_CA_COMMUNICATOR = None
23 # Pipeline model parallel group that the current rank belongs to.
24 _PP_DEVICE_GROUP: Optional[ProcessGroup] = None
25
@@ -47,11 +50,21 @@ _PP_GLOBAL_RANKS: Optional[List[int]] = None
50 _LOCAL_RANK = -1
51
52
53 +def set_custom_all_reduce(enable: bool):
54 + global _ENABLE_CUSTOM_ALL_REDUCE
55 + _ENABLE_CUSTOM_ALL_REDUCE = enable
56 +
57 +
58 def get_tp_pynccl_communicator():
59 global _TP_PYNCCL_COMMUNICATOR
60 return _TP_PYNCCL_COMMUNICATOR
61
62
63 +def get_tp_ca_communicator():
64 + global _TP_CA_COMMUNICATOR
65 + return _TP_CA_COMMUNICATOR
66 +
67 +
68 def get_local_rank():
69 global _LOCAL_RANK
70 return _LOCAL_RANK
@@ -100,6 +113,9 @@ def init_distributed_environment(
113 if torch.cuda.is_available():
114 data = data.to(device=f"cuda:{local_rank}")
115 torch.distributed.all_reduce(data)
116 + if torch.cuda.is_available():
117 + torch.cuda.synchronize()
118 + del data
119
120
121 def initialize_model_parallel(
@@ -149,7 +165,8 @@ def initialize_model_parallel(
165 rank = torch.distributed.get_rank()
166
167 # Build the tensor model-parallel groups.
152 - global _TP_DEVICE_GROUP, _TP_CPU_GROUP, _TP_PYNCCL_COMMUNICATOR
168 + global _TP_DEVICE_GROUP, _TP_CPU_GROUP
169 + global _TP_PYNCCL_COMMUNICATOR, _TP_CA_COMMUNICATOR
170 assert _TP_DEVICE_GROUP is None, (
171 "tensor model parallel group is already initialized")
172 for i in range(num_tensor_model_parallel_groups):
@@ -168,6 +185,15 @@ def initialize_model_parallel(
185 device=_LOCAL_RANK,
186 )
187
188 + # Initialize a custom fast all-reduce implementation.
189 + if _ENABLE_CUSTOM_ALL_REDUCE:
190 + from vllm.distributed.device_communicators.custom_all_reduce import (
191 + CustomAllreduce)
192 + _TP_CA_COMMUNICATOR = CustomAllreduce(
193 + group=_TP_CPU_GROUP,
194 + device=_LOCAL_RANK,
195 + )
196 +
197 # Build the pipeline model-parallel groups.
198 global _PP_DEVICE_GROUP
199 global _PP_GLOBAL_RANKS
vllm/test_utils.py
+8 -9
@@ -6,24 +6,24 @@ from vllm.utils import get_open_port
6
7
8 def init_test_distributed_environment(
9 - pipeline_parallel_size: int,
10 - tensor_parallel_size: int,
9 + tp_size: int,
10 + pp_size: int,
11 rank: int,
12 distributed_init_port: str,
13 local_rank: int = -1,
14 ) -> None:
15 distributed_init_method = f"tcp://localhost:{distributed_init_port}"
16 init_distributed_environment(
17 - world_size=pipeline_parallel_size * tensor_parallel_size,
17 + world_size=pp_size * tp_size,
18 rank=rank,
19 distributed_init_method=distributed_init_method,
20 local_rank=local_rank)
21 - ensure_model_parallel_initialized(tensor_parallel_size,
22 - pipeline_parallel_size)
21 + ensure_model_parallel_initialized(tp_size, pp_size)
22
23
24 def multi_process_tensor_parallel(
26 - tensor_parallel_size: int,
25 + tp_size: int,
26 + pp_size: int,
27 test_target,
28 ) -> None:
29 # Using ray helps debugging the error when it failed
@@ -32,10 +32,9 @@ def multi_process_tensor_parallel(
32
33 distributed_init_port = get_open_port()
34 refs = []
35 - for rank in range(tensor_parallel_size):
35 + for rank in range(tp_size * pp_size):
36 refs.append(
37 - test_target.remote(tensor_parallel_size, rank,
38 - distributed_init_port))
37 + test_target.remote(tp_size, pp_size, rank, distributed_init_port))
38 ray.get(refs)
39
40 ray.shutdown()
vllm/worker/model_runner.py
+4 -11
@@ -12,8 +12,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 ModelConfig, ParallelConfig, SchedulerConfig,
13 VisionLanguageConfig)
14 from vllm.distributed import broadcast_tensor_dict
15 -from vllm.distributed.communication_op import graph_capture_mode
16 -from vllm.distributed.device_communicators import custom_all_reduce
15 +from vllm.distributed.communication_op import graph_capture, graph_mode
16 from vllm.logger import init_logger
17 from vllm.lora.layers import LoRAMapping
18 from vllm.lora.request import LoRARequest
@@ -942,13 +941,7 @@ class ModelRunner:
941 bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
942 ]
943
945 - # NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
946 - # kernel, pynccl, and PyTorch NCCL. When using CUDA graph, we use
947 - # either custom all-reduce kernel or pynccl. When not using CUDA
948 - # graph, we use either custom all-reduce kernel or PyTorch NCCL.
949 - # We always prioritize using custom all-reduce kernel but fall back
950 - # to PyTorch or pynccl if it is disabled or not supported.
951 - with custom_all_reduce.capture():
944 + with graph_capture():
945 # NOTE: Capturing the largest batch size first may help reduce the
946 # memory usage of CUDA graph.
947 for batch_size in reversed(batch_size_capture_list):
@@ -1040,7 +1033,7 @@ class CUDAGraphRunner:
1033 # Run the model once without capturing the graph.
1034 # This is to make sure that the captured graph does not include the
1035 # kernel launches for initial benchmarking (e.g., Triton autotune).
1043 - with graph_capture_mode():
1036 + with graph_mode():
1037 self.model(
1038 input_ids,
1039 positions,
@@ -1055,7 +1048,7 @@ class CUDAGraphRunner:
1048 # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
1049 self._graph = torch.cuda.CUDAGraph()
1050 with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
1058 - with graph_capture_mode():
1051 + with graph_mode():
1052 hidden_states = self.model(
1053 input_ids,
1054 positions,
vllm/worker/worker.py
+4 -7
@@ -11,9 +11,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 VisionLanguageConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
14 - init_distributed_environment)
15 -from vllm.distributed.device_communicators.custom_all_reduce import (
16 - init_custom_ar)
14 + init_distributed_environment,
15 + set_custom_all_reduce)
16 from vllm.lora.request import LoRARequest
17 from vllm.model_executor import set_random_seed
18 from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
@@ -302,16 +301,14 @@ def init_worker_distributed_environment(
301 local_rank: int = -1,
302 ) -> None:
303 """Initialize the distributed environment."""
304 + set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)
305 +
306 init_distributed_environment(parallel_config.world_size, rank,
307 distributed_init_method, local_rank)
308
309 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
310 parallel_config.pipeline_parallel_size)
311
311 - # Initialize a custom fast all-reduce implementation.
312 - if not parallel_config.disable_custom_all_reduce:
313 - init_custom_ar()
314 -
312
313 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
314 # Check if the GPU supports the dtype.