@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e08188081b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] remove graph mode function (#4818)

youkaichao committed May 16, 2024 at 10:59 UTC e08188081be890f72a8d3dda66e7e1ce0a45216c
4 files changed +63 -54
tests/distributed/test_custom_all_reduce.py
+3 -2
@@ -50,7 +50,7 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
50
51 for sz in test_sizes:
52 for dtype in [torch.float32, torch.float16, torch.bfloat16]:
53 - with graph_capture():
53 + with graph_capture() as graph_capture_context:
54 # use integers so result matches NCCL exactly
55 inp1 = torch.randint(1,
56 16, (sz, ),
@@ -62,7 +62,8 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
62 device=torch.cuda.current_device())
63 torch.cuda.synchronize()
64 graph = torch.cuda.CUDAGraph()
65 - with torch.cuda.graph(graph):
65 + with torch.cuda.graph(graph,
66 + stream=graph_capture_context.stream):
67 for i in range(num_communication):
68 out1 = tensor_model_parallel_all_reduce(inp1)
69 # the input buffer is immediately modified to test
tests/distributed/test_pynccl.py
+2 -2
@@ -5,7 +5,7 @@ import pytest
5 import torch
6
7 from vllm.distributed.communication_op import ( # noqa
8 - graph_mode, tensor_model_parallel_all_reduce)
8 + graph_capture, tensor_model_parallel_all_reduce)
9 from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
10 from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
11 from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
@@ -103,7 +103,7 @@ def multiple_tp_with_vllm_worker_fn():
103 device = torch.device(f"cuda:{torch.distributed.get_rank()}")
104 ensure_model_parallel_initialized(2, 2)
105 tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
106 - with graph_mode():
106 + with graph_capture():
107 # two tp groups can communicate independently
108 if torch.distributed.get_rank() in [0, 1]:
109 tensor = tensor_model_parallel_all_reduce(tensor)
vllm/distributed/communication_op.py
+40 -30
@@ -1,5 +1,6 @@
1 from collections import namedtuple
2 from contextlib import contextmanager, nullcontext
3 +from dataclasses import dataclass
4 from typing import Any, Dict, List, Optional, Tuple, Union
5
6 import torch
@@ -13,45 +14,54 @@ from .parallel_state import (get_cpu_world_group,
14 get_tp_pynccl_communicator)
15
16
16 -@contextmanager
17 -def graph_mode():
18 - # In graph mode, we have to be very careful about the collective
19 - # operations. The current status is:
20 - # allreduce \ Mode | Eager | Graph |
21 - # --------------------------------------------
22 - # custom allreduce | enabled | enabled |
23 - # PyNccl | disabled| enabled |
24 - # torch.distributed | enabled | disabled|
25 - #
26 - # Note that custom allreduce will have a runtime check, if the tensor size
27 - # is too large, it will fallback to the next available option.
28 - # In summary: When using CUDA graph, we use
29 - # either custom all-reduce kernel or pynccl. When not using CUDA
30 - # graph, we use either custom all-reduce kernel or PyTorch NCCL.
31 - # We always prioritize using custom all-reduce kernel but fall back
32 - # to PyTorch or pynccl if it is disabled or not supported.
33 - pynccl_comm = get_tp_pynccl_communicator()
34 - if pynccl_comm is None:
35 - context = nullcontext()
36 - else:
37 - context = pynccl_comm.change_state(enable=True,
38 - stream=torch.cuda.current_stream())
39 - with context:
40 - yield
17 +@dataclass
18 +class GraphCaptureContext:
19 + stream: torch.cuda.Stream
20
21
22 @contextmanager
23 def graph_capture():
24 """
46 - `graph_capture` is a context manager which should include the code that
25 + `graph_capture` is a context manager which should surround the code that
26 is capturing the CUDA graph. Its main purpose is to ensure that the
27 some operations will be run after the graph is captured, before the graph
49 - is replayed.
28 + is replayed. It returns a `GraphCaptureContext` object which contains the
29 + necessary data for the graph capture. Currently, it only contains the
30 + stream that the graph capture is running on. This stream is set to the
31 + current CUDA stream when the context manager is entered and reset to the
32 + default stream when the context manager is exited. This is to ensure that
33 + the graph capture is running on a separate stream from the default stream,
34 + in order to explicitly distinguish the kernels to capture
35 + from other kernels possibly launched on background in the default stream.
36 """
37 + stream = torch.cuda.Stream()
38 + graph_capture_context = GraphCaptureContext(stream)
39 ca_comm = get_tp_ca_communicator()
52 - context = nullcontext() if ca_comm is None else ca_comm.capture()
53 - with context:
54 - yield
40 + maybe_ca_context = nullcontext() if ca_comm is None else ca_comm.capture()
41 + with torch.cuda.stream(stream), maybe_ca_context:
42 + # In graph mode, we have to be very careful about the collective
43 + # operations. The current status is:
44 + # allreduce \ Mode | Eager | Graph |
45 + # --------------------------------------------
46 + # custom allreduce | enabled | enabled |
47 + # PyNccl | disabled| enabled |
48 + # torch.distributed | enabled | disabled|
49 + #
50 + # Note that custom allreduce will have a runtime check, if the tensor
51 + # size is too large, it will fallback to the next available option.
52 + # In summary: When using CUDA graph, we use
53 + # either custom all-reduce kernel or pynccl. When not using CUDA
54 + # graph, we use either custom all-reduce kernel or PyTorch NCCL.
55 + # We always prioritize using custom all-reduce kernel but fall back
56 + # to PyTorch or pynccl if it is disabled or not supported.
57 + pynccl_comm = get_tp_pynccl_communicator()
58 + if pynccl_comm is None:
59 + maybe_pynccl_context = nullcontext()
60 + else:
61 + maybe_pynccl_context = pynccl_comm.change_state(
62 + enable=True, stream=torch.cuda.current_stream())
63 + with maybe_pynccl_context:
64 + yield graph_capture_context
65
66
67 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
vllm/worker/model_runner.py
+18 -20
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, ParallelConfig, SchedulerConfig,
11 VisionLanguageConfig)
12 from vllm.distributed import broadcast_tensor_dict
13 -from vllm.distributed.communication_op import graph_capture, graph_mode
13 +from vllm.distributed.communication_op import graph_capture
14 from vllm.logger import init_logger
15 from vllm.lora.layers import LoRAMapping
16 from vllm.lora.request import LoRARequest
@@ -841,7 +841,7 @@ class ModelRunner:
841 bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
842 ]
843
844 - with graph_capture():
844 + with graph_capture() as graph_capture_context:
845 # NOTE: Capturing the largest batch size first may help reduce the
846 # memory usage of CUDA graph.
847 for batch_size in reversed(batch_size_capture_list):
@@ -877,6 +877,7 @@ class ModelRunner:
877 kv_caches,
878 attn_metadata,
879 memory_pool=self.graph_memory_pool,
880 + stream=graph_capture_context.stream,
881 )
882 self.graph_memory_pool = graph_runner.graph.pool()
883 self.graph_runners[batch_size] = graph_runner
@@ -921,15 +922,27 @@ class CUDAGraphRunner:
922 positions: torch.Tensor,
923 kv_caches: List[torch.Tensor],
924 attn_metadata: AttentionMetadata,
924 - memory_pool,
925 + memory_pool: Optional[Tuple[int, int]],
926 + stream: torch.cuda.Stream,
927 **kwargs,
928 ) -> None:
929 assert self._graph is None
930 # Run the model once without capturing the graph.
931 # This is to make sure that the captured graph does not include the
932 # kernel launches for initial benchmarking (e.g., Triton autotune).
931 - with graph_mode():
932 - self.model(
933 + self.model(
934 + input_ids,
935 + positions,
936 + kv_caches,
937 + attn_metadata,
938 + **kwargs,
939 + )
940 + torch.cuda.synchronize()
941 +
942 + # Capture the graph.
943 + self._graph = torch.cuda.CUDAGraph()
944 + with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
945 + hidden_states = self.model(
946 input_ids,
947 positions,
948 kv_caches,
@@ -938,21 +951,6 @@ class CUDAGraphRunner:
951 )
952 torch.cuda.synchronize()
953
941 - # Capture the graph.
942 - # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
943 - # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
944 - self._graph = torch.cuda.CUDAGraph()
945 - with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
946 - with graph_mode():
947 - hidden_states = self.model(
948 - input_ids,
949 - positions,
950 - kv_caches,
951 - attn_metadata,
952 - **kwargs,
953 - )
954 - torch.cuda.synchronize()
955 -
954 # Save the input and output buffers.
955 self.input_buffers = {
956 "input_ids": input_ids,