A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Distributed] refactor custom allreduce to support multiple tp groups (#4754)
youkaichao committed
May 12, 2024 at 17:47 UTC
702bee461f448b0186eb9d673baad29fd923c884
10 files changed
+327
-226
tests/distributed/test_comm_ops.py
+11
-11
@@ -16,7 +16,7 @@ from vllm.test_utils import (init_test_distributed_environment,
16
17
18
@ray.remote(num_gpus=1, max_calls=1)
19
-def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
19
+def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
20
distributed_init_port: str):
21
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
22
# so that each worker can see all the GPUs
@@ -24,12 +24,12 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
24
del os.environ["CUDA_VISIBLE_DEVICES"]
25
device = torch.device(f"cuda:{rank}")
26
torch.cuda.set_device(device)
27
- init_test_distributed_environment(1, tensor_parallel_size, rank,
27
+ init_test_distributed_environment(tp_size, pp_size, rank,
28
distributed_init_port)
29
num_elements = 8
30
all_tensors = [
31
torch.arange(num_elements, dtype=torch.float32, device="cuda") *
32
- (r + 1) for r in range(tensor_parallel_size)
32
+ (r + 1) for r in range(tp_size)
33
]
34
expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
35
t = all_tensors[rank]
@@ -38,7 +38,7 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
38
39
40
@ray.remote(num_gpus=1, max_calls=1)
41
-def all_gather_test_worker(tensor_parallel_size: int, rank: int,
41
+def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
42
distributed_init_port: str):
43
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
44
# so that each worker can see all the GPUs
@@ -46,7 +46,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
46
del os.environ["CUDA_VISIBLE_DEVICES"]
47
device = torch.device(f"cuda:{rank}")
48
torch.cuda.set_device(device)
49
- init_test_distributed_environment(1, tensor_parallel_size, rank,
49
+ init_test_distributed_environment(tp_size, pp_size, rank,
50
distributed_init_port)
51
num_dimensions = 3
52
tensor_size = list(range(2, num_dimensions + 2))
@@ -57,7 +57,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
57
all_tensors = [
58
torch.arange(total_size, dtype=torch.float32,
59
device="cuda").reshape(tensor_size) * (r + 1)
60
- for r in range(tensor_parallel_size)
60
+ for r in range(tp_size)
61
]
62
expected = torch.cat(all_tensors, dim=all_gather_dimension)
63
t = all_tensors[rank]
@@ -66,7 +66,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
66
67
68
@ray.remote(num_gpus=1, max_calls=1)
69
-def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
69
+def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
70
distributed_init_port: str):
71
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
72
# so that each worker can see all the GPUs
@@ -74,7 +74,7 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
74
del os.environ["CUDA_VISIBLE_DEVICES"]
75
device = torch.device(f"cuda:{rank}")
76
torch.cuda.set_device(device)
77
- init_test_distributed_environment(1, tensor_parallel_size, rank,
77
+ init_test_distributed_environment(tp_size, pp_size, rank,
78
distributed_init_port)
79
test_dict = {
80
# device tensor
@@ -106,10 +106,10 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
106
107
@pytest.mark.skipif(torch.cuda.device_count() < 2,
108
reason="Need at least 2 GPUs to run the test.")
109
-@pytest.mark.parametrize("tensor_parallel_size", [2])
109
+@pytest.mark.parametrize("tp_size", [2])
110
@pytest.mark.parametrize("test_target", [
111
all_reduce_test_worker, all_gather_test_worker,
112
broadcast_tensor_dict_test_worker
113
])
114
-def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
115
- multi_process_tensor_parallel(tensor_parallel_size, test_target)
114
+def test_multi_process_tensor_parallel(tp_size, test_target):
115
+ multi_process_tensor_parallel(tp_size, 1, test_target)
tests/distributed/test_custom_all_reduce.py
+58
-29
@@ -6,8 +6,10 @@ import ray
6
import torch
7
import torch.distributed as dist
8
9
-from vllm.distributed import tensor_model_parallel_all_reduce
10
-from vllm.distributed.device_communicators import custom_all_reduce
9
+from vllm.distributed.communication_op import ( # noqa
10
+ graph_capture, tensor_model_parallel_all_reduce)
11
+from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12
+ get_tp_ca_communicator)
13
from vllm.test_utils import (init_test_distributed_environment,
14
multi_process_tensor_parallel)
15
@@ -18,17 +20,36 @@ for i, v in enumerate(test_sizes):
20
21
22
@ray.remote(num_gpus=1, max_calls=1)
21
-def graph_allreduce(world_size, rank, distributed_init_port):
23
+def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
24
del os.environ["CUDA_VISIBLE_DEVICES"]
25
device = torch.device(f"cuda:{rank}")
26
torch.cuda.set_device(device)
25
- init_test_distributed_environment(1, world_size, rank,
27
+ init_test_distributed_environment(tp_size, pp_size, rank,
28
distributed_init_port)
29
28
- custom_all_reduce.init_custom_ar()
30
+ group = get_tensor_model_parallel_group()
31
+
32
+ # A small all_reduce for warmup.
33
+ # this is needed because device communicators might be created lazily
34
+ # (e.g. NCCL). This will ensure that the communicator is initialized
35
+ # before any communication happens, so that this group can be used for
36
+ # graph capture immediately.
37
+ data = torch.zeros(1)
38
+ data = data.to(device=device)
39
+ torch.distributed.all_reduce(data, group=group)
40
+ torch.cuda.synchronize()
41
+ del data
42
+
43
+ # we use the first group to communicate once
44
+ # and the second group to communicate twice
45
+ # and so on
46
+ # this is used to demonstrate that each group can
47
+ # communicate independently
48
+ num_communication = rank // tp_size + 1
49
+
50
for sz in test_sizes:
51
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
31
- with custom_all_reduce.capture():
52
+ with graph_capture():
53
# use integers so result matches NCCL exactly
54
inp1 = torch.randint(1,
55
16, (sz, ),
@@ -41,44 +62,52 @@ def graph_allreduce(world_size, rank, distributed_init_port):
62
torch.cuda.synchronize()
63
graph = torch.cuda.CUDAGraph()
64
with torch.cuda.graph(graph):
44
- out1 = tensor_model_parallel_all_reduce(inp1)
45
- # the input buffer is immediately modified to test
46
- # synchronization
47
- dist.all_reduce(inp1)
48
- out2 = tensor_model_parallel_all_reduce(inp2)
49
- dist.all_reduce(inp2)
65
+ for i in range(num_communication):
66
+ out1 = tensor_model_parallel_all_reduce(inp1)
67
+ # the input buffer is immediately modified to test
68
+ # synchronization
69
+ dist.all_reduce(inp1, group=group)
70
+ out2 = tensor_model_parallel_all_reduce(inp2)
71
+ dist.all_reduce(inp2, group=group)
72
graph.replay()
73
assert torch.allclose(out1, inp1)
74
assert torch.allclose(out2, inp2)
75
76
77
@ray.remote(num_gpus=1, max_calls=1)
56
-def eager_allreduce(world_size, rank, distributed_init_port):
78
+def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
79
del os.environ["CUDA_VISIBLE_DEVICES"]
80
device = torch.device(f"cuda:{rank}")
81
torch.cuda.set_device(device)
60
- init_test_distributed_environment(1, world_size, rank,
82
+ init_test_distributed_environment(tp_size, pp_size, rank,
83
distributed_init_port)
84
85
+ # we use the first group to communicate once
86
+ # and the second group to communicate twice
87
+ # and so on
88
+ # this is used to demonstrate that each group can
89
+ # communicate independently
90
+ num_communication = rank // tp_size + 1
91
sz = 1024
64
- custom_all_reduce.init_custom_ar()
65
- fa = custom_all_reduce.get_handle()
92
+ fa = get_tp_ca_communicator()
93
inp = torch.ones(sz, dtype=torch.float32, device=device)
67
- out = fa.all_reduce_unreg(inp)
68
- assert torch.allclose(out, inp * world_size)
94
+ out = inp
95
+ for _ in range(num_communication):
96
+ out = fa.all_reduce_unreg(out)
97
+ assert torch.allclose(out, inp * (tp_size**num_communication))
98
99
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
71
- out = fa.all_reduce_unreg(inp)
72
- assert torch.allclose(out, inp * world_size)
100
+ out = inp
101
+ for _ in range(num_communication):
102
+ out = fa.all_reduce_unreg(out)
103
+ assert torch.allclose(out, inp * (tp_size**num_communication))
104
105
75
-@pytest.mark.skipif(torch.cuda.device_count() < 2,
76
- reason="Need at least 2 GPUs to run the test.")
77
-@pytest.mark.parametrize("tensor_parallel_size", [2])
106
+@pytest.mark.parametrize("tp_size", [2])
107
+@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
108
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
79
-def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
80
- multi_process_tensor_parallel(tensor_parallel_size, test_target)
81
-
82
-
83
-if __name__ == "__main__":
84
- multi_process_tensor_parallel(2, graph_allreduce)
109
+def test_custom_allreduce(tp_size, pipeline_parallel_size, test_target):
110
+ world_size = tp_size * pipeline_parallel_size
111
+ if world_size > torch.cuda.device_count():
112
+ pytest.skip("Not enough GPUs to run the test.")
113
+ multi_process_tensor_parallel(tp_size, pipeline_parallel_size, test_target)
tests/distributed/test_pynccl.py
+2
-2
@@ -5,7 +5,7 @@ import pytest
5
import torch
6
7
from vllm.distributed.communication_op import ( # noqa
8
- graph_capture_mode, tensor_model_parallel_all_reduce)
8
+ graph_mode, tensor_model_parallel_all_reduce)
9
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
10
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
11
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
@@ -103,7 +103,7 @@ def multiple_tp_with_vllm_worker_fn():
103
device = torch.device(f"cuda:{torch.distributed.get_rank()}")
104
ensure_model_parallel_initialized(2, 2)
105
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
106
- with graph_capture_mode():
106
+ with graph_mode():
107
# two tp groups can communicate independently
108
if torch.distributed.get_rank() in [0, 1]:
109
tensor = tensor_model_parallel_all_reduce(tensor)
vllm/distributed/communication_op.py
+34
-11
@@ -1,5 +1,5 @@
1
from collections import namedtuple
2
-from contextlib import contextmanager
2
+from contextlib import contextmanager, nullcontext
3
from typing import Any, Dict, List, Optional, Tuple, Union
4
5
import torch
@@ -9,12 +9,13 @@ from .parallel_state import (get_cpu_world_group,
9
get_tensor_model_parallel_group,
10
get_tensor_model_parallel_rank,
11
get_tensor_model_parallel_world_size,
12
+ get_tp_ca_communicator,
13
get_tp_pynccl_communicator)
14
15
16
@contextmanager
16
-def graph_capture_mode():
17
- # In graph capture, we have to be very careful about the collective
17
+def graph_mode():
18
+ # In graph mode, we have to be very careful about the collective
19
# operations. The current status is:
20
# allreduce \ Mode | Eager | Graph |
21
# --------------------------------------------
@@ -24,10 +25,32 @@ def graph_capture_mode():
25
#
26
# Note that custom allreduce will have a runtime check, if the tensor size
27
# is too large, it will fallback to the next available option.
28
+ # In summary: When using CUDA graph, we use
29
+ # either custom all-reduce kernel or pynccl. When not using CUDA
30
+ # graph, we use either custom all-reduce kernel or PyTorch NCCL.
31
+ # We always prioritize using custom all-reduce kernel but fall back
32
+ # to PyTorch or pynccl if it is disabled or not supported.
33
pynccl_comm = get_tp_pynccl_communicator()
28
- assert pynccl_comm is not None
29
- with pynccl_comm.change_state(enable=True,
30
- stream=torch.cuda.current_stream()):
34
+ if pynccl_comm is None:
35
+ context = nullcontext()
36
+ else:
37
+ context = pynccl_comm.change_state(enable=True,
38
+ stream=torch.cuda.current_stream())
39
+ with context:
40
+ yield
41
+
42
+
43
+@contextmanager
44
+def graph_capture():
45
+ """
46
+ `graph_capture` is a context manager which should include the code that
47
+ is capturing the CUDA graph. Its main purpose is to ensure that the
48
+ some operations will be run after the graph is captured, before the graph
49
+ is replayed.
50
+ """
51
+ ca_comm = get_tp_ca_communicator()
52
+ context = nullcontext() if ca_comm is None else ca_comm.capture()
53
+ with context:
54
yield
55
56
@@ -43,15 +66,15 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
66
TLDR: always assume this function modifies its input, but use the return
67
value as the output.
68
"""
46
- from vllm.distributed.device_communicators.custom_all_reduce import (
47
- custom_all_reduce)
69
+ ca_comm = get_tp_ca_communicator()
70
71
# Bypass the function if we are using only 1 GPU.
72
if get_tensor_model_parallel_world_size() == 1:
73
return input_
52
- out = custom_all_reduce(input_)
53
- if out is not None:
54
- return out
74
+ if ca_comm is not None:
75
+ out = ca_comm.custom_all_reduce(input_)
76
+ if out is not None:
77
+ return out
78
pynccl_comm = get_tp_pynccl_communicator()
79
if (pynccl_comm is not None and not pynccl_comm.disabled):
80
pynccl_comm.all_reduce(input_)
vllm/distributed/device_communicators/custom_all_reduce.py
+176
-144
@@ -1,154 +1,42 @@
1
from contextlib import contextmanager
2
-from typing import Any, List, Optional
2
+from typing import Any, List, Optional, Union
3
4
import torch
5
import torch.distributed as dist
6
+from torch.distributed import ProcessGroup
7
8
import vllm.envs as envs
9
+from vllm.distributed.parallel_state import (
10
+ get_local_rank, get_tensor_model_parallel_cpu_group)
11
from vllm.logger import init_logger
12
13
try:
14
import pynvml
15
16
from vllm._C import custom_ar
17
+
18
+ @contextmanager
19
+ def _nvml():
20
+ try:
21
+ pynvml.nvmlInit()
22
+ yield
23
+ finally:
24
+ pynvml.nvmlShutdown()
25
+
26
except ImportError:
27
# For AMD GPUs
28
custom_ar = None
29
pynvml = None
30
19
-logger = init_logger(__name__)
31
+ @contextmanager
32
+ def _nvml():
33
+ try:
34
+ yield
35
+ finally:
36
+ pass
37
21
-_CA_HANDLE: Optional["CustomAllreduce"] = None
22
-_IS_CAPTURING = False
23
-_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
24
-
25
-
26
-def init_custom_ar() -> None:
27
- from vllm.distributed import (get_tensor_model_parallel_rank,
28
- get_tensor_model_parallel_world_size)
29
-
30
- global _CA_HANDLE
31
- if _CA_HANDLE is not None:
32
- return
33
- rank = get_tensor_model_parallel_rank()
34
- world_size = get_tensor_model_parallel_world_size()
35
- if world_size == 1:
36
- # No need to initialize custom allreduce for single GPU case.
37
- return
38
-
39
- if world_size not in _SUPPORTED_WORLD_SIZES:
40
- logger.warning(
41
- "Custom allreduce is disabled due to an unsupported world size: "
42
- "%d. Supported world sizes: %s. To silence this warning, specify"
43
- " disable_custom_all_reduce=True explicitly.", world_size,
44
- str(_SUPPORTED_WORLD_SIZES))
45
- return
46
- num_dev = torch.cuda.device_count()
47
- # note: num dev can be larger than world_size if we're only using
48
- # first few GPUs
49
- if num_dev < world_size:
50
- logger.warning(
51
- "Cannot test GPU P2P because not all GPUs are visible to the "
52
- "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
53
- " is set.")
54
- return
55
-
56
- # we only use a subset of GPUs here
57
- # so we only need to check the nvlink connectivity of these GPUs
58
- num_dev = world_size
59
- # test nvlink first, this will filter out most of the cases
60
- # where custom allreduce is not supported
61
- cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
62
- if cuda_visible_devices:
63
- device_ids = list(map(int, cuda_visible_devices.split(",")))
64
- else:
65
- device_ids = list(range(num_dev))
66
- # this checks hardware and driver support for NVLink
67
- full_nvlink = _is_full_nvlink(device_ids)
68
- if world_size > 2 and not full_nvlink:
69
- logger.warning(
70
- "Custom allreduce is disabled because it's not supported on more"
71
- " than two PCIe-only GPUs. To silence this warning, specify"
72
- " disable_custom_all_reduce=True explicitly.")
73
- return
74
- # test P2P capability, this checks software/cudaruntime support
75
- # this is expensive to compute at the first time
76
- # then we cache the result
77
- if not _can_p2p(rank, world_size):
78
- logger.warning(
79
- "Custom allreduce is disabled because your platform lacks GPU P2P"
80
- " capability or P2P test failed. To silence this warning, specify"
81
- " disable_custom_all_reduce=True explicitly.")
82
- return
83
- _CA_HANDLE = CustomAllreduce(rank, world_size, full_nvlink)
84
-
85
-
86
-def begin_capture() -> None:
87
- global _IS_CAPTURING
88
- _IS_CAPTURING = True
89
-
90
-
91
-def end_capture() -> None:
92
- global _IS_CAPTURING
93
- _IS_CAPTURING = False
94
-
95
-
96
-def is_capturing() -> bool:
97
- return _IS_CAPTURING and _CA_HANDLE is not None
98
-
99
-
100
-def get_handle() -> Optional["CustomAllreduce"]:
101
- return _CA_HANDLE
102
-
103
-
104
-def is_initialized() -> bool:
105
- return _CA_HANDLE is not None
106
-
107
-
108
-@contextmanager
109
-def capture():
110
- try:
111
- begin_capture()
112
- yield
113
- finally:
114
- end_capture()
115
- handle = get_handle()
116
- if handle is not None:
117
- handle.register_graph_buffers()
118
-
119
-
120
-def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
121
- ca_handle = get_handle()
122
- # when custom allreduce is disabled, this will be None
123
- if ca_handle is None:
124
- return None
125
- if is_capturing():
126
- if torch.cuda.is_current_stream_capturing():
127
- if ca_handle.should_custom_ar(input):
128
- return ca_handle.all_reduce_reg(input)
129
- else:
130
- if ca_handle.should_custom_ar(input):
131
- # if warm up, mimic the allocation pattern
132
- # since custom allreduce is out-of-place
133
- return torch.empty_like(input)
134
- else:
135
- # note: outside of cuda graph context,
136
- # custom allreduce incurs a cost of cudaMemcpy, which should
137
- # be small(<=1% of overall latency) compared to the performance
138
- # gains of using custom kernels
139
- if ca_handle.should_custom_ar(input):
140
- return ca_handle.all_reduce_unreg(input)
141
-
142
- return None
143
-
144
-
145
-@contextmanager
146
-def _nvml():
147
- try:
148
- pynvml.nvmlInit()
149
- yield
150
- finally:
151
- pynvml.nvmlShutdown()
38
+
39
+logger = init_logger(__name__)
40
41
42
@_nvml()
@@ -188,22 +76,112 @@ def _can_p2p(rank: int, world_size: int) -> bool:
76
77
class CustomAllreduce:
78
79
+ _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
80
+
81
# max_size: max supported allreduce size
82
def __init__(self,
193
- rank,
194
- world_size,
195
- full_nvlink,
83
+ group: Optional[ProcessGroup] = None,
84
+ device: Optional[Union[int, str, torch.device]] = None,
85
max_size=8192 * 1024) -> None:
86
+ """
87
+ Args:
88
+ group: the process group to work on. If None, it will use the
89
+ default process group.
90
+ device: the device to bind the CustomAllreduce to. If None,
91
+ it will be bind to f"cuda:{local_rank}".
92
+ It is the caller's responsibility to make sure each communicator
93
+ is bind to a unique device, and all communicators in this group
94
+ are in the same node.
95
+ """
96
+ self._IS_CAPTURING = False
97
+ self.disabled = True
98
+
99
+ if custom_ar is None:
100
+ # disable because of missing custom allreduce library
101
+ # e.g. in a non-cuda environment
102
+ return
103
+
104
+ group = group or get_tensor_model_parallel_cpu_group()
105
+ self.group = group
106
+
107
+ assert dist.get_backend(group) != dist.Backend.NCCL, (
108
+ "CustomAllreduce should be attached to a non-NCCL group.")
109
+
110
+ rank = dist.get_rank(group=self.group)
111
+ world_size = dist.get_world_size(group=self.group)
112
+ if world_size == 1:
113
+ # No need to initialize custom allreduce for single GPU case.
114
+ return
115
+
116
+ if world_size not in CustomAllreduce._SUPPORTED_WORLD_SIZES:
117
+ logger.warning(
118
+ "Custom allreduce is disabled due to an unsupported world"
119
+ " size: %d. Supported world sizes: %s. To silence this "
120
+ "warning, specify disable_custom_all_reduce=True explicitly.",
121
+ world_size, str(CustomAllreduce._SUPPORTED_WORLD_SIZES))
122
+ return
123
+
124
+ if device is None:
125
+ local_rank = get_local_rank()
126
+ device = torch.device(f"cuda:{local_rank}")
127
+ elif isinstance(device, int):
128
+ device = torch.device(f"cuda:{device}")
129
+ elif isinstance(device, str):
130
+ device = torch.device(device)
131
+ # now `device` is a `torch.device` object
132
+ assert isinstance(device, torch.device)
133
+ self.device = device
134
+
135
+ cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
136
+ if cuda_visible_devices:
137
+ device_ids = list(map(int, cuda_visible_devices.split(",")))
138
+ else:
139
+ device_ids = list(range(torch.cuda.device_count()))
140
+
141
+ physical_device_id = device_ids[device.index]
142
+ tensor = torch.tensor([physical_device_id],
143
+ dtype=torch.int,
144
+ device="cpu")
145
+ gather_list = [
146
+ torch.tensor([0], dtype=torch.int, device="cpu")
147
+ for _ in range(world_size)
148
+ ]
149
+ dist.all_gather(gather_list, tensor, group=self.group)
150
+ physical_device_ids = [t.item() for t in gather_list]
151
+
152
+ # test nvlink first, this will filter out most of the cases
153
+ # where custom allreduce is not supported
154
+ # this checks hardware and driver support for NVLink
155
+ full_nvlink = _is_full_nvlink(physical_device_ids)
156
+ if world_size > 2 and not full_nvlink:
157
+ logger.warning(
158
+ "Custom allreduce is disabled because it's not supported on"
159
+ " more than two PCIe-only GPUs. To silence this warning, "
160
+ "specify disable_custom_all_reduce=True explicitly.")
161
+ return
162
+ # test P2P capability, this checks software/cudaruntime support
163
+ # this is expensive to compute at the first time
164
+ # then we cache the result
165
+ if not _can_p2p(rank, world_size):
166
+ logger.warning(
167
+ "Custom allreduce is disabled because your platform lacks "
168
+ "GPU P2P capability or P2P test failed. To silence this "
169
+ "warning, specify disable_custom_all_reduce=True explicitly.")
170
+ return
171
+
172
+ self.disabled = False
173
# buffers memory are owned by this Python class and passed to C++
174
# meta data composes of two parts: meta data for synchronization
175
# (256 bytes) and a temporary buffer for storing intermediate
176
# allreduce results.
177
self.meta = torch.zeros(custom_ar.meta_size() + max_size,
178
dtype=torch.uint8,
203
- device="cuda")
179
+ device=self.device)
180
# This is a pre-registered IPC buffer. In eager mode, input tensors
181
# are first copied into this buffer before allreduce is performed
206
- self.buffer = torch.empty(max_size, dtype=torch.uint8, device="cuda")
182
+ self.buffer = torch.empty(max_size,
183
+ dtype=torch.uint8,
184
+ device=self.device)
185
# This is a buffer for storing the tuples of pointers pointing to
186
# IPC buffers from all ranks. Each registered tuple has size of
187
# 8*world_size bytes where world_size is at most 8. Allocating 8MB
@@ -211,8 +189,9 @@ class CustomAllreduce:
189
# needs less than 10000 of registered tuples.
190
self.rank_data = torch.empty(8 * 1024 * 1024,
191
dtype=torch.uint8,
214
- device="cuda")
192
+ device=self.device)
193
self.max_size = max_size
194
+ self.rank = rank
195
self.world_size = world_size
196
handles, offsets = self._get_ipc_meta(self.meta)
197
self.full_nvlink = full_nvlink
@@ -221,6 +200,21 @@ class CustomAllreduce:
200
self.full_nvlink)
201
self.register_buffer(self.buffer)
202
203
+ @contextmanager
204
+ def capture(self):
205
+ """
206
+ The main responsibility of this context manager is the
207
+ `register_graph_buffers` call at the end of the context.
208
+ It records all the buffer addresses used in the CUDA graph.
209
+ """
210
+ try:
211
+ self._IS_CAPTURING = True
212
+ yield
213
+ finally:
214
+ self._IS_CAPTURING = False
215
+ if not self.disabled:
216
+ self.register_graph_buffers()
217
+
218
def _get_ipc_meta(self, inp: torch.Tensor):
219
data = inp.untyped_storage()._share_cuda_()
220
shard_data = (
@@ -230,14 +224,29 @@ class CustomAllreduce:
224
return self._gather_ipc_meta(shard_data)
225
226
def _gather_ipc_meta(self, shard_data):
233
- all_data: List[Optional[Any]] = [None] * self.world_size
234
- dist.all_gather_object(all_data, shard_data)
227
+ # Note: don't use `[[None]] * self.world_size` here
228
+ # because it will create a list of the same reference
229
+ all_data: List[Optional[Any]] = [[None]
230
+ for i in range(self.world_size)]
231
+ all_data[self.rank][0] = shard_data
232
+
233
+ ranks = dist.get_process_group_ranks(group=self.group)
234
+ ranks.sort()
235
+ for i, rank in enumerate(ranks):
236
+ dist.broadcast_object_list(all_data[i],
237
+ src=rank,
238
+ group=self.group,
239
+ device="cpu")
240
+
241
+ # we cannot directly use `dist.all_gather_object` here
242
+ # because it is incompatible with `gloo` backend under inference mode.
243
+ # see https://github.com/pytorch/pytorch/issues/126032 for details.
244
245
handles = []
246
offsets = []
247
for i in range(len(all_data)):
239
- handles.append(all_data[i][0]) # type: ignore
240
- offsets.append(all_data[i][1]) # type: ignore
248
+ handles.append(all_data[i][0][0]) # type: ignore
249
+ offsets.append(all_data[i][0][1]) # type: ignore
250
return handles, offsets
251
252
def register_buffer(self, inp: torch.Tensor):
@@ -269,8 +278,31 @@ class CustomAllreduce:
278
custom_ar.all_reduce_unreg(self._ptr, inp, self.buffer, out)
279
return out
280
281
+ def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
282
+ # when custom allreduce is disabled, this will be None
283
+ if self.disabled:
284
+ return None
285
+ if self._IS_CAPTURING:
286
+ if torch.cuda.is_current_stream_capturing():
287
+ if self.should_custom_ar(input):
288
+ return self.all_reduce_reg(input)
289
+ else:
290
+ if self.should_custom_ar(input):
291
+ # if warm up, mimic the allocation pattern
292
+ # since custom allreduce is out-of-place
293
+ return torch.empty_like(input)
294
+ else:
295
+ # note: outside of cuda graph context,
296
+ # custom allreduce incurs a cost of cudaMemcpy, which should
297
+ # be small(<=1% of overall latency) compared to the performance
298
+ # gains of using custom kernels
299
+ if self.should_custom_ar(input):
300
+ return self.all_reduce_unreg(input)
301
+
302
+ return None
303
+
304
def close(self):
273
- if self._ptr:
305
+ if not self.disabled and self._ptr:
306
custom_ar.dispose(self._ptr)
307
self._ptr = 0
308
vllm/distributed/device_communicators/pynccl.py
+3
-1
@@ -96,8 +96,10 @@ class PyNcclCommunicator:
96
self.stream = torch.cuda.Stream()
97
98
# A small all_reduce for warmup.
99
- self.all_reduce(torch.zeros(1, device=device))
99
+ data = torch.zeros(1, device=device)
100
+ self.all_reduce(data)
101
self.stream.synchronize()
102
+ del data
103
104
# by default it is disabled, e.g. in profiling models and prefill phase.
105
# to use it, use under `with obj.change_state(enable=True)`, usually
vllm/distributed/parallel_state.py
+27
-1
@@ -13,10 +13,13 @@ from vllm.logger import init_logger
13
14
logger = init_logger(__name__)
15
16
+_ENABLE_CUSTOM_ALL_REDUCE = True
17
+
18
# Tensor model parallel group that the current rank belongs to.
19
_TP_DEVICE_GROUP: Optional[ProcessGroup] = None
20
_TP_CPU_GROUP: Optional[ProcessGroup] = None
21
_TP_PYNCCL_COMMUNICATOR = None
22
+_TP_CA_COMMUNICATOR = None
23
# Pipeline model parallel group that the current rank belongs to.
24
_PP_DEVICE_GROUP: Optional[ProcessGroup] = None
25
@@ -47,11 +50,21 @@ _PP_GLOBAL_RANKS: Optional[List[int]] = None
50
_LOCAL_RANK = -1
51
52
53
+def set_custom_all_reduce(enable: bool):
54
+ global _ENABLE_CUSTOM_ALL_REDUCE
55
+ _ENABLE_CUSTOM_ALL_REDUCE = enable
56
+
57
+
58
def get_tp_pynccl_communicator():
59
global _TP_PYNCCL_COMMUNICATOR
60
return _TP_PYNCCL_COMMUNICATOR
61
62
63
+def get_tp_ca_communicator():
64
+ global _TP_CA_COMMUNICATOR
65
+ return _TP_CA_COMMUNICATOR
66
+
67
+
68
def get_local_rank():
69
global _LOCAL_RANK
70
return _LOCAL_RANK
@@ -100,6 +113,9 @@ def init_distributed_environment(
113
if torch.cuda.is_available():
114
data = data.to(device=f"cuda:{local_rank}")
115
torch.distributed.all_reduce(data)
116
+ if torch.cuda.is_available():
117
+ torch.cuda.synchronize()
118
+ del data
119
120
121
def initialize_model_parallel(
@@ -149,7 +165,8 @@ def initialize_model_parallel(
165
rank = torch.distributed.get_rank()
166
167
# Build the tensor model-parallel groups.
152
- global _TP_DEVICE_GROUP, _TP_CPU_GROUP, _TP_PYNCCL_COMMUNICATOR
168
+ global _TP_DEVICE_GROUP, _TP_CPU_GROUP
169
+ global _TP_PYNCCL_COMMUNICATOR, _TP_CA_COMMUNICATOR
170
assert _TP_DEVICE_GROUP is None, (
171
"tensor model parallel group is already initialized")
172
for i in range(num_tensor_model_parallel_groups):
@@ -168,6 +185,15 @@ def initialize_model_parallel(
185
device=_LOCAL_RANK,
186
)
187
188
+ # Initialize a custom fast all-reduce implementation.
189
+ if _ENABLE_CUSTOM_ALL_REDUCE:
190
+ from vllm.distributed.device_communicators.custom_all_reduce import (
191
+ CustomAllreduce)
192
+ _TP_CA_COMMUNICATOR = CustomAllreduce(
193
+ group=_TP_CPU_GROUP,
194
+ device=_LOCAL_RANK,
195
+ )
196
+
197
# Build the pipeline model-parallel groups.
198
global _PP_DEVICE_GROUP
199
global _PP_GLOBAL_RANKS
vllm/test_utils.py
+8
-9
@@ -6,24 +6,24 @@ from vllm.utils import get_open_port
6
7
8
def init_test_distributed_environment(
9
- pipeline_parallel_size: int,
10
- tensor_parallel_size: int,
9
+ tp_size: int,
10
+ pp_size: int,
11
rank: int,
12
distributed_init_port: str,
13
local_rank: int = -1,
14
) -> None:
15
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
16
init_distributed_environment(
17
- world_size=pipeline_parallel_size * tensor_parallel_size,
17
+ world_size=pp_size * tp_size,
18
rank=rank,
19
distributed_init_method=distributed_init_method,
20
local_rank=local_rank)
21
- ensure_model_parallel_initialized(tensor_parallel_size,
22
- pipeline_parallel_size)
21
+ ensure_model_parallel_initialized(tp_size, pp_size)
22
23
24
def multi_process_tensor_parallel(
26
- tensor_parallel_size: int,
25
+ tp_size: int,
26
+ pp_size: int,
27
test_target,
28
) -> None:
29
# Using ray helps debugging the error when it failed
@@ -32,10 +32,9 @@ def multi_process_tensor_parallel(
32
33
distributed_init_port = get_open_port()
34
refs = []
35
- for rank in range(tensor_parallel_size):
35
+ for rank in range(tp_size * pp_size):
36
refs.append(
37
- test_target.remote(tensor_parallel_size, rank,
38
- distributed_init_port))
37
+ test_target.remote(tp_size, pp_size, rank, distributed_init_port))
38
ray.get(refs)
39
40
ray.shutdown()
vllm/worker/model_runner.py
+4
-11
@@ -12,8 +12,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
ModelConfig, ParallelConfig, SchedulerConfig,
13
VisionLanguageConfig)
14
from vllm.distributed import broadcast_tensor_dict
15
-from vllm.distributed.communication_op import graph_capture_mode
16
-from vllm.distributed.device_communicators import custom_all_reduce
15
+from vllm.distributed.communication_op import graph_capture, graph_mode
16
from vllm.logger import init_logger
17
from vllm.lora.layers import LoRAMapping
18
from vllm.lora.request import LoRARequest
@@ -942,13 +941,7 @@ class ModelRunner:
941
bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
942
]
943
945
- # NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
946
- # kernel, pynccl, and PyTorch NCCL. When using CUDA graph, we use
947
- # either custom all-reduce kernel or pynccl. When not using CUDA
948
- # graph, we use either custom all-reduce kernel or PyTorch NCCL.
949
- # We always prioritize using custom all-reduce kernel but fall back
950
- # to PyTorch or pynccl if it is disabled or not supported.
951
- with custom_all_reduce.capture():
944
+ with graph_capture():
945
# NOTE: Capturing the largest batch size first may help reduce the
946
# memory usage of CUDA graph.
947
for batch_size in reversed(batch_size_capture_list):
@@ -1040,7 +1033,7 @@ class CUDAGraphRunner:
1033
# Run the model once without capturing the graph.
1034
# This is to make sure that the captured graph does not include the
1035
# kernel launches for initial benchmarking (e.g., Triton autotune).
1043
- with graph_capture_mode():
1036
+ with graph_mode():
1037
self.model(
1038
input_ids,
1039
positions,
@@ -1055,7 +1048,7 @@ class CUDAGraphRunner:
1048
# https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
1049
self._graph = torch.cuda.CUDAGraph()
1050
with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
1058
- with graph_capture_mode():
1051
+ with graph_mode():
1052
hidden_states = self.model(
1053
input_ids,
1054
positions,
vllm/worker/worker.py
+4
-7
@@ -11,9 +11,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
VisionLanguageConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
14
- init_distributed_environment)
15
-from vllm.distributed.device_communicators.custom_all_reduce import (
16
- init_custom_ar)
14
+ init_distributed_environment,
15
+ set_custom_all_reduce)
16
from vllm.lora.request import LoRARequest
17
from vllm.model_executor import set_random_seed
18
from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
@@ -302,16 +301,14 @@ def init_worker_distributed_environment(
301
local_rank: int = -1,
302
) -> None:
303
"""Initialize the distributed environment."""
304
+ set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)
305
+
306
init_distributed_environment(parallel_config.world_size, rank,
307
distributed_init_method, local_rank)
308
309
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
310
parallel_config.pipeline_parallel_size)
311
311
- # Initialize a custom fast all-reduce implementation.
312
- if not parallel_config.disable_custom_all_reduce:
313
- init_custom_ar()
314
-
312
313
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
314
# Check if the GPU supports the dtype.