@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a463c333dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Use CuPy for CUDA graphs (#2811)

Woosuk Kwon committed Feb 13, 2024 at 11:32 UTC a463c333dd7905519141abe4f61b63ccc6b739a9
8 files changed +246 -22
requirements.txt
+1
@@ -12,3 +12,4 @@ pydantic >= 2.0 # Required for OpenAI server.
12 aioprometheus[starlette]
13 pynvml == 11.5.0
14 triton >= 2.1.0
15 +cupy-cuda12x == 12.3.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead.
vllm/engine/llm_engine.py
+1 -1
@@ -283,7 +283,7 @@ class LLMEngine:
283 is_driver_worker=True,
284 )
285
286 - self._run_workers("init_model")
286 + self._run_workers("init_model", cupy_port=get_open_port())
287 self._run_workers(
288 "load_model",
289 max_concurrent_workers=self.parallel_config.
vllm/model_executor/parallel_utils/communication_op.py
+9 -4
@@ -1,14 +1,15 @@
1 from collections import namedtuple
2 from typing import Any, Dict, List, Optional, Union
3
4 -from torch.distributed import ProcessGroup
5 -
4 import torch
5 +from torch.distributed import ProcessGroup
6
7 +from vllm.model_executor.parallel_utils import cupy_utils
8 from vllm.model_executor.parallel_utils.parallel_state import (
9 get_tensor_model_parallel_rank,
10 get_tensor_model_parallel_world_size,
11 get_tensor_model_parallel_group,
12 + is_cupy_nccl_enabled_for_all_reduce,
13 )
14 from vllm.model_executor.parallel_utils.custom_all_reduce import custom_all_reduce
15
@@ -31,8 +32,12 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
32 out = custom_all_reduce(input_)
33 if out is not None:
34 return out
34 - torch.distributed.all_reduce(input_,
35 - group=get_tensor_model_parallel_group())
35 + if is_cupy_nccl_enabled_for_all_reduce():
36 + # TODO: support multiple parallel groups.
37 + cupy_utils.all_reduce(input_)
38 + else:
39 + torch.distributed.all_reduce(input_,
40 + group=get_tensor_model_parallel_group())
41 return input_
42
43
vllm/model_executor/parallel_utils/cupy_utils.py new
+130
@@ -0,0 +1,130 @@
1 +"""CuPy utilities for all-reduce.
2 +
3 +We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4 +CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5 +CUDA graphs.
6 +
7 +NOTE: We use CuPy 12.3 since CuPy 13.0 does not support Python 3.8.
8 +TODO: Remove this file when torch.distributed.all_reduce is fixed.
9 +"""
10 +import contextlib
11 +
12 +import torch
13 +from torch.distributed import ReduceOp
14 +
15 +try:
16 + import cupy
17 + from cupy.cuda import nccl
18 + from cupyx.distributed import NCCLBackend
19 +except ImportError as e:
20 + cupy = e
21 + nccl = None
22 +
23 + class NCCLBackend:
24 + ...
25 +
26 +
27 +_OP_MAPPING = {
28 + ReduceOp.SUM: "sum",
29 + ReduceOp.PRODUCT: "prod",
30 + ReduceOp.MIN: "min",
31 + ReduceOp.MAX: "max",
32 +}
33 +
34 +
35 +class NCCLBackendWithBFloat16(NCCLBackend):
36 + # This is enough to add bfloat16 support for most operations,
37 + # but broadcast will fail (will require changes in compiled
38 + # cupy code).
39 + def _get_nccl_dtype_and_count(self, array, count=None):
40 + nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
41 + torch_dtype = getattr(array, "_torch_dtype", None)
42 + if torch_dtype is torch.bfloat16:
43 + nccl_dtype = nccl.NCCL_BFLOAT16
44 + return nccl_dtype, count
45 +
46 + def barrier(self) -> None:
47 + raise RuntimeError(
48 + "Currently, CuPy NCCL barrier is not supported since the TCP "
49 + "store is immediately stopped after the initialization.")
50 +
51 +
52 +_NCCL_BACKEND = None
53 +_WORLD_SIZE = 0
54 +
55 +
56 +def is_initialized() -> bool:
57 + """Returns whether the NCCL backend is initialized."""
58 + return _NCCL_BACKEND is not None
59 +
60 +
61 +@contextlib.contextmanager
62 +def set_cupy_stream(stream: torch.cuda.Stream):
63 + """Set the cuda stream for communication"""
64 + cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
65 + stream.device_index)
66 + with cupy_stream:
67 + yield
68 +
69 +
70 +def init_process_group(world_size: int, rank: int, host: str,
71 + port: int) -> None:
72 + """Initializes the CuPy NCCL backend.
73 +
74 + # TODO: handle NCCL timeouts.
75 + """
76 + assert not is_initialized()
77 +
78 + if isinstance(cupy, Exception):
79 + raise ImportError(
80 + "NCCLBackend is not available. Please install cupy.") from cupy
81 +
82 + # TODO(woosuk): Create TP and PP process groups for CuPy.
83 + global _NCCL_BACKEND
84 + global _WORLD_SIZE
85 + assert world_size > 0, f"{world_size=} should be a positive integer"
86 + assert 0 <= rank < world_size, (
87 + f"{rank=} should be a integer between [0, {world_size})")
88 +
89 + cupy.cuda.runtime.setDevice(torch.cuda.current_device())
90 + _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
91 + _WORLD_SIZE = world_size
92 +
93 + # Stop the TCP store to prevent the deadlock issues at termination time.
94 + # FIXME(woosuk): This is hacky. Find a more robust solution.
95 + if rank == 0 and hasattr(_NCCL_BACKEND, "_store"):
96 + _NCCL_BACKEND._store.stop()
97 +
98 +
99 +def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
100 + """All-reduces the input tensor across the process group."""
101 + assert input_.is_cuda, f"{input_} should be a cuda tensor"
102 + # Hack to support bfloat16
103 + torch_dtype = input_.dtype
104 + if torch_dtype is torch.bfloat16:
105 + # We need to view as float16, otherwise
106 + # cupy will fail. This will not change
107 + # the underlying data.
108 + input_ = input_.view(torch.float16)
109 + cupy_input = cupy.asarray(input_)
110 + cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
111 + _NCCL_BACKEND.all_reduce(in_array=cupy_input,
112 + out_array=cupy_input,
113 + op=_OP_MAPPING[op])
114 +
115 +
116 +def destroy_process_group() -> None:
117 + """Destroys the NCCL backend."""
118 + global _NCCL_BACKEND
119 + global _WORLD_SIZE
120 + _NCCL_BACKEND = None
121 + _WORLD_SIZE = 0
122 +
123 +
124 +def get_world_size() -> int:
125 + """Returns the world size."""
126 + return _WORLD_SIZE
127 +
128 +
129 +def get_nccl_backend():
130 + return _NCCL_BACKEND
vllm/model_executor/parallel_utils/parallel_state.py
+37
@@ -3,9 +3,12 @@
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 """Tensor and pipeline parallel groups."""
6 +import contextlib
7
8 import torch
9
10 +from vllm.model_executor.parallel_utils import cupy_utils
11 +
12 # Tensor model parallel group that the current rank belongs to.
13 _TENSOR_MODEL_PARALLEL_GROUP = None
14 # Pipeline model parallel group that the current rank belongs to.
@@ -206,3 +209,37 @@ def destroy_model_parallel():
209 _PIPELINE_MODEL_PARALLEL_GROUP = None
210 global _PIPELINE_GLOBAL_RANKS
211 _PIPELINE_GLOBAL_RANKS = None
212 +
213 + # Destroy the cupy states if any.
214 + cupy_utils.destroy_process_group()
215 +
216 +
217 +# Whether to use cupy for nccl all reduce.
218 +# We use cupy for all reduce when using CUDA graph, because torch.distributed
219 +# is not well supported by CUDA graph.
220 +_ENABLE_CUPY_FOR_ALL_REDUCE = False
221 +
222 +
223 +@contextlib.contextmanager
224 +def with_cupy_nccl_for_all_reduce():
225 + """use CuPy nccl instead of torch.distributed for all reduce"""
226 + tp_size = get_tensor_model_parallel_world_size()
227 + if tp_size == 1:
228 + # No-op.
229 + # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
230 + yield
231 + else:
232 + global _ENABLE_CUPY_FOR_ALL_REDUCE
233 + old = _ENABLE_CUPY_FOR_ALL_REDUCE
234 + _ENABLE_CUPY_FOR_ALL_REDUCE = True
235 +
236 + stream = torch.cuda.current_stream()
237 + with cupy_utils.set_cupy_stream(stream):
238 + yield
239 + _ENABLE_CUPY_FOR_ALL_REDUCE = old
240 +
241 +
242 +def is_cupy_nccl_enabled_for_all_reduce():
243 + """check if CuPy nccl is enabled for all reduce"""
244 + global _ENABLE_CUPY_FOR_ALL_REDUCE
245 + return _ENABLE_CUPY_FOR_ALL_REDUCE
vllm/test_utils.py
+5 -2
@@ -15,8 +15,11 @@ def init_test_distributed_environment(
15 tensor_parallel_size,
16 worker_use_ray=True)
17 distributed_init_method = f"tcp://localhost:{distributed_init_port}"
18 - init_distributed_environment(parallel_config, rank,
19 - distributed_init_method)
18 + init_distributed_environment(
19 + parallel_config,
20 + rank,
21 + cupy_port=None,
22 + distributed_init_method=distributed_init_method)
23
24
25 def multi_process_tensor_parallel(
vllm/worker/model_runner.py
+39 -13
@@ -5,11 +5,15 @@ import numpy as np
5 import torch
6 import torch.nn as nn
7
8 -from vllm.config import DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
8 +from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
9 + SchedulerConfig)
10 from vllm.logger import init_logger
11 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
12 from vllm.model_executor.parallel_utils.communication_op import (
13 broadcast_tensor_dict)
14 +from vllm.model_executor.parallel_utils.cupy_utils import get_nccl_backend
15 +from vllm.model_executor.parallel_utils.parallel_state import (
16 + with_cupy_nccl_for_all_reduce)
17 from vllm.model_executor.parallel_utils import custom_all_reduce
18 from vllm.sampling_params import SamplingParams, SamplingType
19 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
@@ -644,6 +648,10 @@ class ModelRunner:
648
649 @torch.inference_mode()
650 def capture_model(self, kv_caches: List[KVCache]) -> None:
651 + # NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
652 + # deleted before the CUDA graphs.
653 + self.cupy_nccl_backend = get_nccl_backend()
654 +
655 assert not self.model_config.enforce_eager
656 logger.info("Capturing the model for CUDA graphs. This may lead to "
657 "unexpected consequences if the model is not static. To "
@@ -674,6 +682,12 @@ class ModelRunner:
682
683 # NOTE: Capturing the largest batch size first may help reduce the
684 # memory usage of CUDA graph.
685 + # NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
686 + # kernel, CuPy NCCL, and PyTorch NCCL. When using CUDA graph, we use
687 + # either custom all-reduce kernel or CuPy NCCL. When not using CUDA
688 + # graph, we use either custom all-reduce kernel or PyTorch NCCL.
689 + # We always prioritize using custom all-reduce kernel but fall back
690 + # to PyTorch or CuPy NCCL if it is disabled or not supported.
691 with custom_all_reduce.capture():
692 for batch_size in reversed(batch_size_capture_list):
693 # Create dummy input_metadata.
@@ -713,6 +727,14 @@ class ModelRunner:
727 # This usually takes < 10 seconds.
728 logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
729
730 + def __del__(self) -> None:
731 + # Delete the CUDA graphs before deleting the CuPy NCCL communicator.
732 + # NOTE(woosuk): This is necessary because otherwise deadlocks can
733 + # happen.
734 + # FIXME(woosuk): This is a bit hacky. Find a more robust solution.
735 + self.graph_runners.clear()
736 + self.cupy_nccl_backend = None
737 +
738
739 class CUDAGraphRunner:
740
@@ -734,18 +756,8 @@ class CUDAGraphRunner:
756 # Run the model once without capturing the graph.
757 # This is to make sure that the captured graph does not include the
758 # kernel launches for initial benchmarking (e.g., Triton autotune).
737 - self.model(
738 - input_ids,
739 - positions,
740 - kv_caches,
741 - input_metadata,
742 - )
743 - torch.cuda.synchronize()
744 -
745 - # Capture the graph.
746 - self.graph = torch.cuda.CUDAGraph()
747 - with torch.cuda.graph(self.graph, pool=memory_pool):
748 - hidden_states = self.model(
759 + with with_cupy_nccl_for_all_reduce():
760 + self.model(
761 input_ids,
762 positions,
763 kv_caches,
@@ -753,6 +765,20 @@ class CUDAGraphRunner:
765 )
766 torch.cuda.synchronize()
767
768 + # Capture the graph.
769 + # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
770 + # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
771 + self.graph = torch.cuda.CUDAGraph()
772 + with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
773 + with with_cupy_nccl_for_all_reduce():
774 + hidden_states = self.model(
775 + input_ids,
776 + positions,
777 + kv_caches,
778 + input_metadata,
779 + )
780 + torch.cuda.synchronize()
781 +
782 # Save the input and output buffers.
783 self.input_buffers = {
784 "input_ids": input_ids,
vllm/worker/worker.py
+24 -2
@@ -9,6 +9,7 @@ import torch.distributed
9 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10 ParallelConfig, SchedulerConfig, LoRAConfig)
11 from vllm.model_executor import set_random_seed
12 +from vllm.model_executor.parallel_utils import cupy_utils
13 from vllm.model_executor.parallel_utils.communication_op import (
14 broadcast_tensor_dict)
15 from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
@@ -67,7 +68,7 @@ class Worker:
68 self.cache_events = None
69 self.gpu_cache = None
70
70 - def init_model(self) -> None:
71 + def init_model(self, cupy_port: Optional[int] = None) -> None:
72 if self.device_config.device.type == "cuda":
73 # torch.distributed.all_reduce does not free the input tensor until
74 # the synchronization point. This causes the memory usage to grow
@@ -88,7 +89,7 @@ class Worker:
89 f"Not support device type: {self.device_config.device}")
90 # Initialize the distributed environment.
91 init_distributed_environment(self.parallel_config, self.rank,
91 - self.distributed_init_method)
92 + cupy_port, self.distributed_init_method)
93 if not self.parallel_config.disable_custom_all_reduce:
94 init_custom_ar()
95 # Initialize the model.
@@ -233,6 +234,7 @@ class Worker:
234 def init_distributed_environment(
235 parallel_config: ParallelConfig,
236 rank: int,
237 + cupy_port: Optional[int],
238 distributed_init_method: Optional[str] = None,
239 ) -> None:
240 """Initialize the distributed environment."""
@@ -255,8 +257,28 @@ def init_distributed_environment(
257 init_method=distributed_init_method,
258 )
259
260 + if cupy_utils.is_initialized():
261 + cupy_world_size = cupy_utils.get_world_size()
262 + if cupy_world_size != parallel_config.world_size:
263 + raise RuntimeError(
264 + "cupy.distributed is already initialized but the cupy world "
265 + "size does not match parallel_config.world_size "
266 + f"({cupy_world_size} vs. {parallel_config.world_size}).")
267 + elif parallel_config.world_size > 1 and cupy_port is not None:
268 + # NOTE(woosuk): We don't initialize CuPy process group when world size
269 + # is 1.
270 + # TODO(woosuk): Support multi-node connection.
271 + cupy_utils.init_process_group(
272 + world_size=parallel_config.world_size,
273 + rank=rank,
274 + host="localhost",
275 + port=cupy_port,
276 + )
277 +
278 # A small all_reduce for warmup.
279 torch.distributed.all_reduce(torch.zeros(1).cuda())
280 + if cupy_utils.is_initialized():
281 + cupy_utils.all_reduce(torch.zeros(1).cuda())
282 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
283 parallel_config.pipeline_parallel_size)
284