A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
custom allreduce + torch.compile (#10121)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Sage Moore committed
Nov 26, 2024 at 00:00 UTC
9a88f897993a83fad79d1bf6b95595be25a8d68a
6 files changed
+59
-101
docs/source/getting_started/debugging.rst
-1
@@ -86,7 +86,6 @@ If GPU/CPU communication cannot be established, you can use the following Python
86
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
87
88
pynccl = PyNcclCommunicator(group=gloo_group, device=local_rank)
89
- pynccl.disabled = False
89
90
s = torch.cuda.Stream()
91
with torch.cuda.stream(s):
tests/distributed/test_pynccl.py
+6
-9
@@ -60,7 +60,7 @@ def worker_fn():
60
tensor = torch.ones(16, 1024, 1024,
61
dtype=torch.float32).cuda(pynccl_comm.rank)
62
with pynccl_comm.change_state(enable=True):
63
- pynccl_comm.all_reduce(tensor)
63
+ tensor = pynccl_comm.all_reduce(tensor)
64
result = tensor.mean().cpu().item()
65
assert result == pynccl_comm.world_size
66
@@ -84,12 +84,12 @@ def multiple_allreduce_worker_fn():
84
with pynccl_comm.change_state(enable=True):
85
# two groups can communicate independently
86
if torch.distributed.get_rank() in [0, 1]:
87
- pynccl_comm.all_reduce(tensor)
88
- pynccl_comm.all_reduce(tensor)
87
+ tensor = pynccl_comm.all_reduce(tensor)
88
+ tensor = pynccl_comm.all_reduce(tensor)
89
result = tensor.mean().cpu().item()
90
assert result == 4
91
else:
92
- pynccl_comm.all_reduce(tensor)
92
+ tensor = pynccl_comm.all_reduce(tensor)
93
result = tensor.mean().cpu().item()
94
assert result == 2
95
@@ -140,14 +140,11 @@ def worker_fn_with_cudagraph():
140
with torch.cuda.graph(
141
graph, stream=pynccl_comm.stream), pynccl_comm.change_state(
142
enable=True):
143
- # operation during the graph capture is recorded but not executed
144
- # see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#creating-a-graph-using-stream-capture # noqa
145
- pynccl_comm.all_reduce(a)
143
+ a_out = pynccl_comm.all_reduce(a)
144
pynccl_comm.stream.synchronize()
147
- assert a.mean().cpu().item() == pynccl_comm.world_size**0
145
graph.replay()
146
pynccl_comm.stream.synchronize()
150
- assert a.mean().cpu().item() == pynccl_comm.world_size**1
147
+ assert a_out.mean().cpu().item() == pynccl_comm.world_size**1
148
149
150
@worker_fn_wrapper
tests/distributed/test_utils.py
-2
@@ -70,14 +70,12 @@ def gpu_worker(rank, WORLD_SIZE, port1, port2):
70
rank=rank,
71
world_size=WORLD_SIZE)
72
pynccl1 = PyNcclCommunicator(pg1, device=rank)
73
- pynccl1.disabled = False
73
if rank <= 2:
74
pg2 = StatelessProcessGroup.create(host="127.0.0.1",
75
port=port2,
76
rank=rank,
77
world_size=3)
78
pynccl2 = PyNcclCommunicator(pg2, device=rank)
80
- pynccl2.disabled = False
79
data = torch.tensor([rank]).cuda()
80
pynccl1.all_reduce(data)
81
pg1.barrier()
vllm/distributed/device_communicators/pynccl.py
+13
-13
@@ -106,30 +106,30 @@ class PyNcclCommunicator:
106
self.stream.synchronize()
107
del data
108
109
- # by default it is disabled, e.g. in profiling models and prefill phase.
110
- # to use it, use under `with obj.change_state(enable=True)`, usually
111
- # when we are using CUDA graph.
112
- self.disabled = True
113
-
109
def all_reduce(self,
115
- tensor: torch.Tensor,
110
+ in_tensor: torch.Tensor,
111
op: ReduceOp = ReduceOp.SUM,
117
- stream=None):
112
+ stream=None) -> torch.Tensor:
113
if self.disabled:
119
- return
114
+ return None
115
# nccl communicator created on a specific device
116
# will only work on tensors on the same device
117
# otherwise it will cause "illegal memory access"
123
- assert tensor.device == self.device, (
118
+ assert in_tensor.device == self.device, (
119
f"this nccl communicator is created to work on {self.device}, "
125
- f"but the input tensor is on {tensor.device}")
120
+ f"but the input tensor is on {in_tensor.device}")
121
+
122
+ out_tensor = torch.empty_like(in_tensor)
123
+
124
if stream is None:
125
stream = self.stream
128
- self.nccl.ncclAllReduce(buffer_type(tensor.data_ptr()),
129
- buffer_type(tensor.data_ptr()), tensor.numel(),
130
- ncclDataTypeEnum.from_torch(tensor.dtype),
126
+ self.nccl.ncclAllReduce(buffer_type(in_tensor.data_ptr()),
127
+ buffer_type(out_tensor.data_ptr()),
128
+ in_tensor.numel(),
129
+ ncclDataTypeEnum.from_torch(in_tensor.dtype),
130
ncclRedOpTypeEnum.from_torch(op), self.comm,
131
cudaStream_t(stream.cuda_stream))
132
+ return out_tensor
133
134
def all_gather(self,
135
output_tensor: torch.Tensor,
vllm/distributed/parallel_state.py
+36
-74
@@ -96,42 +96,24 @@ def _register_group(group: "GroupCoordinator") -> None:
96
_groups[group.unique_name] = weakref.ref(group)
97
98
99
-if supports_custom_op():
100
-
101
- def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None:
102
- assert group_name in _groups, f"Group {group_name} is not found."
103
- group = _groups[group_name]()
104
- if group is None:
105
- raise ValueError(f"Group {group_name} is destroyed.")
106
- group._all_reduce_in_place(tensor)
107
-
108
- def inplace_all_reduce_fake(tensor: torch.Tensor, group_name: str) -> None:
109
- return
99
+def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
100
+ assert group_name in _groups, f"Group {group_name} is not found."
101
+ group = _groups[group_name]()
102
+ if group is None:
103
+ raise ValueError(f"Group {group_name} is destroyed.")
104
+ return group._all_reduce_out_place(tensor)
105
111
- direct_register_custom_op(
112
- op_name="inplace_all_reduce",
113
- op_func=inplace_all_reduce,
114
- mutates_args=["tensor"],
115
- fake_impl=inplace_all_reduce_fake,
116
- )
106
118
- def outplace_all_reduce(tensor: torch.Tensor,
119
- group_name: str) -> torch.Tensor:
120
- assert group_name in _groups, f"Group {group_name} is not found."
121
- group = _groups[group_name]()
122
- if group is None:
123
- raise ValueError(f"Group {group_name} is destroyed.")
124
- return group._all_reduce_out_place(tensor)
107
+def all_reduce_fake(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
108
+ return torch.empty_like(tensor)
109
126
- def outplace_all_reduce_fake(tensor: torch.Tensor,
127
- group_name: str) -> torch.Tensor:
128
- return torch.empty_like(tensor)
110
111
+if supports_custom_op():
112
direct_register_custom_op(
131
- op_name="outplace_all_reduce",
132
- op_func=outplace_all_reduce,
113
+ op_name="all_reduce",
114
+ op_func=all_reduce,
115
mutates_args=[],
134
- fake_impl=outplace_all_reduce_fake,
116
+ fake_impl=all_reduce_fake,
117
)
118
119
@@ -317,30 +299,13 @@ class GroupCoordinator:
299
stream.wait_stream(curr_stream)
300
301
with torch.cuda.stream(stream), maybe_ca_context:
320
- # In graph mode, we have to be very careful about the collective
321
- # operations. The current status is:
322
- # allreduce \ Mode | Eager | Graph |
323
- # --------------------------------------------
324
- # custom allreduce | enabled | enabled |
325
- # PyNccl | disabled| enabled |
326
- # torch.distributed | enabled | disabled|
327
- #
328
- # Note that custom allreduce will have a runtime check, if the
329
- # tensor size is too large, it will fallback to the next
330
- # available option.
331
- # In summary: When using CUDA graph, we use
332
- # either custom all-reduce kernel or pynccl. When not using
333
- # CUDA graph, we use either custom all-reduce kernel or
334
- # PyTorch NCCL. We always prioritize using custom all-reduce
335
- # kernel but fall back to PyTorch or pynccl if it is
336
- # disabled or not supported.
302
pynccl_comm = self.pynccl_comm
303
maybe_pynccl_context: Any
304
if not pynccl_comm:
305
maybe_pynccl_context = nullcontext()
306
else:
307
maybe_pynccl_context = pynccl_comm.change_state(
343
- enable=True, stream=torch.cuda.current_stream())
308
+ stream=torch.cuda.current_stream())
309
with maybe_pynccl_context:
310
yield graph_capture_context
311
@@ -356,8 +321,8 @@ class GroupCoordinator:
321
coordinator.
322
323
In addition, PyTorch custom ops do not support mutation or returning
359
- a new tensor in the same op. So we need to figure out if the op is
360
- in-place or out-of-place ahead of time.
324
+ a new tensor in the same op. So we always make the all-reduce operation
325
+ out-of-place.
326
"""
327
# Bypass the function if we are using only 1 GPU.
328
if self.world_size == 1:
@@ -368,10 +333,6 @@ class GroupCoordinator:
333
ipex.distributed.all_reduce(input_, group=self.device_group)
334
return input_
335
371
- if not supports_custom_op():
372
- self._all_reduce_in_place(input_)
373
- return input_
374
-
336
if self.tpu_communicator is not None and \
337
not self.tpu_communicator.disabled:
338
# TPU handles Dynamo with its own logic.
@@ -385,30 +346,31 @@ class GroupCoordinator:
346
not self.xpu_communicator.disabled:
347
return self.xpu_communicator.all_reduce(input_)
348
388
- if self.ca_comm is not None and \
389
- not self.ca_comm.disabled and \
390
- self.ca_comm.should_custom_ar(input_):
391
- return torch.ops.vllm.outplace_all_reduce(
392
- input_, group_name=self.unique_name)
393
- else:
394
- torch.ops.vllm.inplace_all_reduce(input_,
395
- group_name=self.unique_name)
396
- return input_
349
+ return torch.ops.vllm.all_reduce(input_, group_name=self.unique_name)
350
351
def _all_reduce_out_place(self, input_: torch.Tensor) -> torch.Tensor:
352
+ # always try custom allreduce first,
353
+ # and then pynccl.
354
ca_comm = self.ca_comm
400
- assert ca_comm is not None
401
- assert not ca_comm.disabled
402
- out = ca_comm.custom_all_reduce(input_)
403
- assert out is not None
404
- return out
405
-
406
- def _all_reduce_in_place(self, input_: torch.Tensor) -> None:
355
+ if ca_comm is not None and not ca_comm.disabled and \
356
+ ca_comm.should_custom_ar(input_):
357
+ out = ca_comm.custom_all_reduce(input_)
358
+ assert out is not None
359
+ return out
360
pynccl_comm = self.pynccl_comm
408
- if (pynccl_comm is not None and not pynccl_comm.disabled):
409
- pynccl_comm.all_reduce(input_)
410
- else:
411
- torch.distributed.all_reduce(input_, group=self.device_group)
361
+ assert pynccl_comm is not None
362
+ # TODO: pynccl should not use `stream=`
363
+ # it can just always use the current stream.
364
+ out = pynccl_comm.all_reduce(input_,
365
+ stream=torch.cuda.current_stream())
366
+ if out is None:
367
+ # fall back to the default all-reduce using PyTorch.
368
+ # this usually happens during testing.
369
+ # when we run the model, allreduce only happens for the TP
370
+ # group, where we always have either custom allreduce or pynccl.
371
+ out = input_.clone()
372
+ torch.distributed.all_reduce(out, group=self.device_group)
373
+ return out
374
375
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
376
world_size = self.world_size
vllm/v1/worker/gpu_model_runner.py
+4
-2
@@ -10,6 +10,7 @@ import torch.nn as nn
10
11
from vllm.compilation.compile_context import set_compile_context
12
from vllm.config import CompilationLevel, VllmConfig
13
+from vllm.distributed.parallel_state import graph_capture
14
from vllm.forward_context import set_forward_context
15
from vllm.inputs import INPUT_REGISTRY, InputRegistry
16
from vllm.logger import init_logger
@@ -570,8 +571,9 @@ class GPUModelRunner:
571
# Trigger CUDA graph capture for specific shapes.
572
# Capture the large shapes first so that the smaller shapes
573
# can reuse the memory pool allocated for the large shapes.
573
- for num_tokens in reversed(self.cudagraph_batch_sizes):
574
- self._dummy_run(self.model, num_tokens, self.kv_caches)
574
+ with graph_capture():
575
+ for num_tokens in reversed(self.cudagraph_batch_sizes):
576
+ self._dummy_run(self.model, num_tokens, self.kv_caches)
577
578
end_time = time.perf_counter()
579
end_free_gpu_memory = torch.cuda.mem_get_info()[0]