A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Use CuPy for CUDA graphs (#2811)
Woosuk Kwon committed
Feb 13, 2024 at 11:32 UTC
a463c333dd7905519141abe4f61b63ccc6b739a9
8 files changed
+246
-22
requirements.txt
+1
@@ -12,3 +12,4 @@ pydantic >= 2.0 # Required for OpenAI server.
12
aioprometheus[starlette]
13
pynvml == 11.5.0
14
triton >= 2.1.0
15
+cupy-cuda12x == 12.3.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead.
vllm/engine/llm_engine.py
+1
-1
@@ -283,7 +283,7 @@ class LLMEngine:
283
is_driver_worker=True,
284
)
285
286
- self._run_workers("init_model")
286
+ self._run_workers("init_model", cupy_port=get_open_port())
287
self._run_workers(
288
"load_model",
289
max_concurrent_workers=self.parallel_config.
vllm/model_executor/parallel_utils/communication_op.py
+9
-4
@@ -1,14 +1,15 @@
1
from collections import namedtuple
2
from typing import Any, Dict, List, Optional, Union
3
4
-from torch.distributed import ProcessGroup
5
-
4
import torch
5
+from torch.distributed import ProcessGroup
6
7
+from vllm.model_executor.parallel_utils import cupy_utils
8
from vllm.model_executor.parallel_utils.parallel_state import (
9
get_tensor_model_parallel_rank,
10
get_tensor_model_parallel_world_size,
11
get_tensor_model_parallel_group,
12
+ is_cupy_nccl_enabled_for_all_reduce,
13
)
14
from vllm.model_executor.parallel_utils.custom_all_reduce import custom_all_reduce
15
@@ -31,8 +32,12 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
32
out = custom_all_reduce(input_)
33
if out is not None:
34
return out
34
- torch.distributed.all_reduce(input_,
35
- group=get_tensor_model_parallel_group())
35
+ if is_cupy_nccl_enabled_for_all_reduce():
36
+ # TODO: support multiple parallel groups.
37
+ cupy_utils.all_reduce(input_)
38
+ else:
39
+ torch.distributed.all_reduce(input_,
40
+ group=get_tensor_model_parallel_group())
41
return input_
42
43
vllm/model_executor/parallel_utils/cupy_utils.py
new
+130
@@ -0,0 +1,130 @@
1
+"""CuPy utilities for all-reduce.
2
+
3
+We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4
+CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5
+CUDA graphs.
6
+
7
+NOTE: We use CuPy 12.3 since CuPy 13.0 does not support Python 3.8.
8
+TODO: Remove this file when torch.distributed.all_reduce is fixed.
9
+"""
10
+import contextlib
11
+
12
+import torch
13
+from torch.distributed import ReduceOp
14
+
15
+try:
16
+ import cupy
17
+ from cupy.cuda import nccl
18
+ from cupyx.distributed import NCCLBackend
19
+except ImportError as e:
20
+ cupy = e
21
+ nccl = None
22
+
23
+ class NCCLBackend:
24
+ ...
25
+
26
+
27
+_OP_MAPPING = {
28
+ ReduceOp.SUM: "sum",
29
+ ReduceOp.PRODUCT: "prod",
30
+ ReduceOp.MIN: "min",
31
+ ReduceOp.MAX: "max",
32
+}
33
+
34
+
35
+class NCCLBackendWithBFloat16(NCCLBackend):
36
+ # This is enough to add bfloat16 support for most operations,
37
+ # but broadcast will fail (will require changes in compiled
38
+ # cupy code).
39
+ def _get_nccl_dtype_and_count(self, array, count=None):
40
+ nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
41
+ torch_dtype = getattr(array, "_torch_dtype", None)
42
+ if torch_dtype is torch.bfloat16:
43
+ nccl_dtype = nccl.NCCL_BFLOAT16
44
+ return nccl_dtype, count
45
+
46
+ def barrier(self) -> None:
47
+ raise RuntimeError(
48
+ "Currently, CuPy NCCL barrier is not supported since the TCP "
49
+ "store is immediately stopped after the initialization.")
50
+
51
+
52
+_NCCL_BACKEND = None
53
+_WORLD_SIZE = 0
54
+
55
+
56
+def is_initialized() -> bool:
57
+ """Returns whether the NCCL backend is initialized."""
58
+ return _NCCL_BACKEND is not None
59
+
60
+
61
+@contextlib.contextmanager
62
+def set_cupy_stream(stream: torch.cuda.Stream):
63
+ """Set the cuda stream for communication"""
64
+ cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
65
+ stream.device_index)
66
+ with cupy_stream:
67
+ yield
68
+
69
+
70
+def init_process_group(world_size: int, rank: int, host: str,
71
+ port: int) -> None:
72
+ """Initializes the CuPy NCCL backend.
73
+
74
+ # TODO: handle NCCL timeouts.
75
+ """
76
+ assert not is_initialized()
77
+
78
+ if isinstance(cupy, Exception):
79
+ raise ImportError(
80
+ "NCCLBackend is not available. Please install cupy.") from cupy
81
+
82
+ # TODO(woosuk): Create TP and PP process groups for CuPy.
83
+ global _NCCL_BACKEND
84
+ global _WORLD_SIZE
85
+ assert world_size > 0, f"{world_size=} should be a positive integer"
86
+ assert 0 <= rank < world_size, (
87
+ f"{rank=} should be a integer between [0, {world_size})")
88
+
89
+ cupy.cuda.runtime.setDevice(torch.cuda.current_device())
90
+ _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
91
+ _WORLD_SIZE = world_size
92
+
93
+ # Stop the TCP store to prevent the deadlock issues at termination time.
94
+ # FIXME(woosuk): This is hacky. Find a more robust solution.
95
+ if rank == 0 and hasattr(_NCCL_BACKEND, "_store"):
96
+ _NCCL_BACKEND._store.stop()
97
+
98
+
99
+def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
100
+ """All-reduces the input tensor across the process group."""
101
+ assert input_.is_cuda, f"{input_} should be a cuda tensor"
102
+ # Hack to support bfloat16
103
+ torch_dtype = input_.dtype
104
+ if torch_dtype is torch.bfloat16:
105
+ # We need to view as float16, otherwise
106
+ # cupy will fail. This will not change
107
+ # the underlying data.
108
+ input_ = input_.view(torch.float16)
109
+ cupy_input = cupy.asarray(input_)
110
+ cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
111
+ _NCCL_BACKEND.all_reduce(in_array=cupy_input,
112
+ out_array=cupy_input,
113
+ op=_OP_MAPPING[op])
114
+
115
+
116
+def destroy_process_group() -> None:
117
+ """Destroys the NCCL backend."""
118
+ global _NCCL_BACKEND
119
+ global _WORLD_SIZE
120
+ _NCCL_BACKEND = None
121
+ _WORLD_SIZE = 0
122
+
123
+
124
+def get_world_size() -> int:
125
+ """Returns the world size."""
126
+ return _WORLD_SIZE
127
+
128
+
129
+def get_nccl_backend():
130
+ return _NCCL_BACKEND
vllm/model_executor/parallel_utils/parallel_state.py
+37
@@ -3,9 +3,12 @@
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
+import contextlib
7
8
import torch
9
10
+from vllm.model_executor.parallel_utils import cupy_utils
11
+
12
# Tensor model parallel group that the current rank belongs to.
13
_TENSOR_MODEL_PARALLEL_GROUP = None
14
# Pipeline model parallel group that the current rank belongs to.
@@ -206,3 +209,37 @@ def destroy_model_parallel():
209
_PIPELINE_MODEL_PARALLEL_GROUP = None
210
global _PIPELINE_GLOBAL_RANKS
211
_PIPELINE_GLOBAL_RANKS = None
212
+
213
+ # Destroy the cupy states if any.
214
+ cupy_utils.destroy_process_group()
215
+
216
+
217
+# Whether to use cupy for nccl all reduce.
218
+# We use cupy for all reduce when using CUDA graph, because torch.distributed
219
+# is not well supported by CUDA graph.
220
+_ENABLE_CUPY_FOR_ALL_REDUCE = False
221
+
222
+
223
+@contextlib.contextmanager
224
+def with_cupy_nccl_for_all_reduce():
225
+ """use CuPy nccl instead of torch.distributed for all reduce"""
226
+ tp_size = get_tensor_model_parallel_world_size()
227
+ if tp_size == 1:
228
+ # No-op.
229
+ # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
230
+ yield
231
+ else:
232
+ global _ENABLE_CUPY_FOR_ALL_REDUCE
233
+ old = _ENABLE_CUPY_FOR_ALL_REDUCE
234
+ _ENABLE_CUPY_FOR_ALL_REDUCE = True
235
+
236
+ stream = torch.cuda.current_stream()
237
+ with cupy_utils.set_cupy_stream(stream):
238
+ yield
239
+ _ENABLE_CUPY_FOR_ALL_REDUCE = old
240
+
241
+
242
+def is_cupy_nccl_enabled_for_all_reduce():
243
+ """check if CuPy nccl is enabled for all reduce"""
244
+ global _ENABLE_CUPY_FOR_ALL_REDUCE
245
+ return _ENABLE_CUPY_FOR_ALL_REDUCE
vllm/test_utils.py
+5
-2
@@ -15,8 +15,11 @@ def init_test_distributed_environment(
15
tensor_parallel_size,
16
worker_use_ray=True)
17
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
18
- init_distributed_environment(parallel_config, rank,
19
- distributed_init_method)
18
+ init_distributed_environment(
19
+ parallel_config,
20
+ rank,
21
+ cupy_port=None,
22
+ distributed_init_method=distributed_init_method)
23
24
25
def multi_process_tensor_parallel(
vllm/worker/model_runner.py
+39
-13
@@ -5,11 +5,15 @@ import numpy as np
5
import torch
6
import torch.nn as nn
7
8
-from vllm.config import DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
8
+from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
9
+ SchedulerConfig)
10
from vllm.logger import init_logger
11
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
12
from vllm.model_executor.parallel_utils.communication_op import (
13
broadcast_tensor_dict)
14
+from vllm.model_executor.parallel_utils.cupy_utils import get_nccl_backend
15
+from vllm.model_executor.parallel_utils.parallel_state import (
16
+ with_cupy_nccl_for_all_reduce)
17
from vllm.model_executor.parallel_utils import custom_all_reduce
18
from vllm.sampling_params import SamplingParams, SamplingType
19
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
@@ -644,6 +648,10 @@ class ModelRunner:
648
649
@torch.inference_mode()
650
def capture_model(self, kv_caches: List[KVCache]) -> None:
651
+ # NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
652
+ # deleted before the CUDA graphs.
653
+ self.cupy_nccl_backend = get_nccl_backend()
654
+
655
assert not self.model_config.enforce_eager
656
logger.info("Capturing the model for CUDA graphs. This may lead to "
657
"unexpected consequences if the model is not static. To "
@@ -674,6 +682,12 @@ class ModelRunner:
682
683
# NOTE: Capturing the largest batch size first may help reduce the
684
# memory usage of CUDA graph.
685
+ # NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
686
+ # kernel, CuPy NCCL, and PyTorch NCCL. When using CUDA graph, we use
687
+ # either custom all-reduce kernel or CuPy NCCL. When not using CUDA
688
+ # graph, we use either custom all-reduce kernel or PyTorch NCCL.
689
+ # We always prioritize using custom all-reduce kernel but fall back
690
+ # to PyTorch or CuPy NCCL if it is disabled or not supported.
691
with custom_all_reduce.capture():
692
for batch_size in reversed(batch_size_capture_list):
693
# Create dummy input_metadata.
@@ -713,6 +727,14 @@ class ModelRunner:
727
# This usually takes < 10 seconds.
728
logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
729
730
+ def __del__(self) -> None:
731
+ # Delete the CUDA graphs before deleting the CuPy NCCL communicator.
732
+ # NOTE(woosuk): This is necessary because otherwise deadlocks can
733
+ # happen.
734
+ # FIXME(woosuk): This is a bit hacky. Find a more robust solution.
735
+ self.graph_runners.clear()
736
+ self.cupy_nccl_backend = None
737
+
738
739
class CUDAGraphRunner:
740
@@ -734,18 +756,8 @@ class CUDAGraphRunner:
756
# Run the model once without capturing the graph.
757
# This is to make sure that the captured graph does not include the
758
# kernel launches for initial benchmarking (e.g., Triton autotune).
737
- self.model(
738
- input_ids,
739
- positions,
740
- kv_caches,
741
- input_metadata,
742
- )
743
- torch.cuda.synchronize()
744
-
745
- # Capture the graph.
746
- self.graph = torch.cuda.CUDAGraph()
747
- with torch.cuda.graph(self.graph, pool=memory_pool):
748
- hidden_states = self.model(
759
+ with with_cupy_nccl_for_all_reduce():
760
+ self.model(
761
input_ids,
762
positions,
763
kv_caches,
@@ -753,6 +765,20 @@ class CUDAGraphRunner:
765
)
766
torch.cuda.synchronize()
767
768
+ # Capture the graph.
769
+ # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
770
+ # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
771
+ self.graph = torch.cuda.CUDAGraph()
772
+ with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
773
+ with with_cupy_nccl_for_all_reduce():
774
+ hidden_states = self.model(
775
+ input_ids,
776
+ positions,
777
+ kv_caches,
778
+ input_metadata,
779
+ )
780
+ torch.cuda.synchronize()
781
+
782
# Save the input and output buffers.
783
self.input_buffers = {
784
"input_ids": input_ids,
vllm/worker/worker.py
+24
-2
@@ -9,6 +9,7 @@ import torch.distributed
9
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10
ParallelConfig, SchedulerConfig, LoRAConfig)
11
from vllm.model_executor import set_random_seed
12
+from vllm.model_executor.parallel_utils import cupy_utils
13
from vllm.model_executor.parallel_utils.communication_op import (
14
broadcast_tensor_dict)
15
from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
@@ -67,7 +68,7 @@ class Worker:
68
self.cache_events = None
69
self.gpu_cache = None
70
70
- def init_model(self) -> None:
71
+ def init_model(self, cupy_port: Optional[int] = None) -> None:
72
if self.device_config.device.type == "cuda":
73
# torch.distributed.all_reduce does not free the input tensor until
74
# the synchronization point. This causes the memory usage to grow
@@ -88,7 +89,7 @@ class Worker:
89
f"Not support device type: {self.device_config.device}")
90
# Initialize the distributed environment.
91
init_distributed_environment(self.parallel_config, self.rank,
91
- self.distributed_init_method)
92
+ cupy_port, self.distributed_init_method)
93
if not self.parallel_config.disable_custom_all_reduce:
94
init_custom_ar()
95
# Initialize the model.
@@ -233,6 +234,7 @@ class Worker:
234
def init_distributed_environment(
235
parallel_config: ParallelConfig,
236
rank: int,
237
+ cupy_port: Optional[int],
238
distributed_init_method: Optional[str] = None,
239
) -> None:
240
"""Initialize the distributed environment."""
@@ -255,8 +257,28 @@ def init_distributed_environment(
257
init_method=distributed_init_method,
258
)
259
260
+ if cupy_utils.is_initialized():
261
+ cupy_world_size = cupy_utils.get_world_size()
262
+ if cupy_world_size != parallel_config.world_size:
263
+ raise RuntimeError(
264
+ "cupy.distributed is already initialized but the cupy world "
265
+ "size does not match parallel_config.world_size "
266
+ f"({cupy_world_size} vs. {parallel_config.world_size}).")
267
+ elif parallel_config.world_size > 1 and cupy_port is not None:
268
+ # NOTE(woosuk): We don't initialize CuPy process group when world size
269
+ # is 1.
270
+ # TODO(woosuk): Support multi-node connection.
271
+ cupy_utils.init_process_group(
272
+ world_size=parallel_config.world_size,
273
+ rank=rank,
274
+ host="localhost",
275
+ port=cupy_port,
276
+ )
277
+
278
# A small all_reduce for warmup.
279
torch.distributed.all_reduce(torch.zeros(1).cuda())
280
+ if cupy_utils.is_initialized():
281
+ cupy_utils.all_reduce(torch.zeros(1).cuda())
282
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
283
parallel_config.pipeline_parallel_size)
284