A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Distributed] remove graph mode function (#4818)
youkaichao committed
May 16, 2024 at 10:59 UTC
e08188081be890f72a8d3dda66e7e1ce0a45216c
4 files changed
+63
-54
tests/distributed/test_custom_all_reduce.py
+3
-2
@@ -50,7 +50,7 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
50
51
for sz in test_sizes:
52
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
53
- with graph_capture():
53
+ with graph_capture() as graph_capture_context:
54
# use integers so result matches NCCL exactly
55
inp1 = torch.randint(1,
56
16, (sz, ),
@@ -62,7 +62,8 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
62
device=torch.cuda.current_device())
63
torch.cuda.synchronize()
64
graph = torch.cuda.CUDAGraph()
65
- with torch.cuda.graph(graph):
65
+ with torch.cuda.graph(graph,
66
+ stream=graph_capture_context.stream):
67
for i in range(num_communication):
68
out1 = tensor_model_parallel_all_reduce(inp1)
69
# the input buffer is immediately modified to test
tests/distributed/test_pynccl.py
+2
-2
@@ -5,7 +5,7 @@ import pytest
5
import torch
6
7
from vllm.distributed.communication_op import ( # noqa
8
- graph_mode, tensor_model_parallel_all_reduce)
8
+ graph_capture, tensor_model_parallel_all_reduce)
9
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
10
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
11
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
@@ -103,7 +103,7 @@ def multiple_tp_with_vllm_worker_fn():
103
device = torch.device(f"cuda:{torch.distributed.get_rank()}")
104
ensure_model_parallel_initialized(2, 2)
105
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
106
- with graph_mode():
106
+ with graph_capture():
107
# two tp groups can communicate independently
108
if torch.distributed.get_rank() in [0, 1]:
109
tensor = tensor_model_parallel_all_reduce(tensor)
vllm/distributed/communication_op.py
+40
-30
@@ -1,5 +1,6 @@
1
from collections import namedtuple
2
from contextlib import contextmanager, nullcontext
3
+from dataclasses import dataclass
4
from typing import Any, Dict, List, Optional, Tuple, Union
5
6
import torch
@@ -13,45 +14,54 @@ from .parallel_state import (get_cpu_world_group,
14
get_tp_pynccl_communicator)
15
16
16
-@contextmanager
17
-def graph_mode():
18
- # In graph mode, we have to be very careful about the collective
19
- # operations. The current status is:
20
- # allreduce \ Mode | Eager | Graph |
21
- # --------------------------------------------
22
- # custom allreduce | enabled | enabled |
23
- # PyNccl | disabled| enabled |
24
- # torch.distributed | enabled | disabled|
25
- #
26
- # Note that custom allreduce will have a runtime check, if the tensor size
27
- # is too large, it will fallback to the next available option.
28
- # In summary: When using CUDA graph, we use
29
- # either custom all-reduce kernel or pynccl. When not using CUDA
30
- # graph, we use either custom all-reduce kernel or PyTorch NCCL.
31
- # We always prioritize using custom all-reduce kernel but fall back
32
- # to PyTorch or pynccl if it is disabled or not supported.
33
- pynccl_comm = get_tp_pynccl_communicator()
34
- if pynccl_comm is None:
35
- context = nullcontext()
36
- else:
37
- context = pynccl_comm.change_state(enable=True,
38
- stream=torch.cuda.current_stream())
39
- with context:
40
- yield
17
+@dataclass
18
+class GraphCaptureContext:
19
+ stream: torch.cuda.Stream
20
21
22
@contextmanager
23
def graph_capture():
24
"""
46
- `graph_capture` is a context manager which should include the code that
25
+ `graph_capture` is a context manager which should surround the code that
26
is capturing the CUDA graph. Its main purpose is to ensure that the
27
some operations will be run after the graph is captured, before the graph
49
- is replayed.
28
+ is replayed. It returns a `GraphCaptureContext` object which contains the
29
+ necessary data for the graph capture. Currently, it only contains the
30
+ stream that the graph capture is running on. This stream is set to the
31
+ current CUDA stream when the context manager is entered and reset to the
32
+ default stream when the context manager is exited. This is to ensure that
33
+ the graph capture is running on a separate stream from the default stream,
34
+ in order to explicitly distinguish the kernels to capture
35
+ from other kernels possibly launched on background in the default stream.
36
"""
37
+ stream = torch.cuda.Stream()
38
+ graph_capture_context = GraphCaptureContext(stream)
39
ca_comm = get_tp_ca_communicator()
52
- context = nullcontext() if ca_comm is None else ca_comm.capture()
53
- with context:
54
- yield
40
+ maybe_ca_context = nullcontext() if ca_comm is None else ca_comm.capture()
41
+ with torch.cuda.stream(stream), maybe_ca_context:
42
+ # In graph mode, we have to be very careful about the collective
43
+ # operations. The current status is:
44
+ # allreduce \ Mode | Eager | Graph |
45
+ # --------------------------------------------
46
+ # custom allreduce | enabled | enabled |
47
+ # PyNccl | disabled| enabled |
48
+ # torch.distributed | enabled | disabled|
49
+ #
50
+ # Note that custom allreduce will have a runtime check, if the tensor
51
+ # size is too large, it will fallback to the next available option.
52
+ # In summary: When using CUDA graph, we use
53
+ # either custom all-reduce kernel or pynccl. When not using CUDA
54
+ # graph, we use either custom all-reduce kernel or PyTorch NCCL.
55
+ # We always prioritize using custom all-reduce kernel but fall back
56
+ # to PyTorch or pynccl if it is disabled or not supported.
57
+ pynccl_comm = get_tp_pynccl_communicator()
58
+ if pynccl_comm is None:
59
+ maybe_pynccl_context = nullcontext()
60
+ else:
61
+ maybe_pynccl_context = pynccl_comm.change_state(
62
+ enable=True, stream=torch.cuda.current_stream())
63
+ with maybe_pynccl_context:
64
+ yield graph_capture_context
65
66
67
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
vllm/worker/model_runner.py
+18
-20
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, ParallelConfig, SchedulerConfig,
11
VisionLanguageConfig)
12
from vllm.distributed import broadcast_tensor_dict
13
-from vllm.distributed.communication_op import graph_capture, graph_mode
13
+from vllm.distributed.communication_op import graph_capture
14
from vllm.logger import init_logger
15
from vllm.lora.layers import LoRAMapping
16
from vllm.lora.request import LoRARequest
@@ -841,7 +841,7 @@ class ModelRunner:
841
bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
842
]
843
844
- with graph_capture():
844
+ with graph_capture() as graph_capture_context:
845
# NOTE: Capturing the largest batch size first may help reduce the
846
# memory usage of CUDA graph.
847
for batch_size in reversed(batch_size_capture_list):
@@ -877,6 +877,7 @@ class ModelRunner:
877
kv_caches,
878
attn_metadata,
879
memory_pool=self.graph_memory_pool,
880
+ stream=graph_capture_context.stream,
881
)
882
self.graph_memory_pool = graph_runner.graph.pool()
883
self.graph_runners[batch_size] = graph_runner
@@ -921,15 +922,27 @@ class CUDAGraphRunner:
922
positions: torch.Tensor,
923
kv_caches: List[torch.Tensor],
924
attn_metadata: AttentionMetadata,
924
- memory_pool,
925
+ memory_pool: Optional[Tuple[int, int]],
926
+ stream: torch.cuda.Stream,
927
**kwargs,
928
) -> None:
929
assert self._graph is None
930
# Run the model once without capturing the graph.
931
# This is to make sure that the captured graph does not include the
932
# kernel launches for initial benchmarking (e.g., Triton autotune).
931
- with graph_mode():
932
- self.model(
933
+ self.model(
934
+ input_ids,
935
+ positions,
936
+ kv_caches,
937
+ attn_metadata,
938
+ **kwargs,
939
+ )
940
+ torch.cuda.synchronize()
941
+
942
+ # Capture the graph.
943
+ self._graph = torch.cuda.CUDAGraph()
944
+ with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
945
+ hidden_states = self.model(
946
input_ids,
947
positions,
948
kv_caches,
@@ -938,21 +951,6 @@ class CUDAGraphRunner:
951
)
952
torch.cuda.synchronize()
953
941
- # Capture the graph.
942
- # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
943
- # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
944
- self._graph = torch.cuda.CUDAGraph()
945
- with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
946
- with graph_mode():
947
- hidden_states = self.model(
948
- input_ids,
949
- positions,
950
- kv_caches,
951
- attn_metadata,
952
- **kwargs,
953
- )
954
- torch.cuda.synchronize()
955
-
954
# Save the input and output buffers.
955
self.input_buffers = {
956
"input_ids": input_ids,