A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Remove dependency on CuPy (#2152)
Woosuk Kwon committed
Dec 17, 2023 at 01:49 UTC
c3372e87bed990510e4ae0b39f151a34dea24f8b
7 files changed
+19
-208
requirements.txt
-1
@@ -12,4 +12,3 @@ fastapi
12
uvicorn[standard]
13
pydantic == 1.10.13 # Required for OpenAI server.
14
aioprometheus[starlette]
15
-cupy-cuda12x # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead. # FIXME: Fix this in setup.py.
vllm/engine/llm_engine.py
+1
-2
@@ -17,7 +17,7 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
17
SequenceOutput, SequenceStatus)
18
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
19
get_tokenizer)
20
-from vllm.utils import Counter, get_open_port
20
+from vllm.utils import Counter
21
22
if ray:
23
from ray.air.util.torch_dist import init_torch_dist_process_group
@@ -190,7 +190,6 @@ class LLMEngine:
190
))
191
self._run_workers(
192
"init_model",
193
- cupy_port=get_open_port(),
193
get_all_outputs=True,
194
)
195
self._run_workers(
vllm/model_executor/parallel_utils/communication_op.py
+2
-8
@@ -1,10 +1,8 @@
1
import torch
2
3
-from vllm.model_executor.parallel_utils import cupy_utils
3
from vllm.model_executor.parallel_utils.parallel_state import (
4
get_tensor_model_parallel_world_size,
5
get_tensor_model_parallel_group,
7
- is_custom_nccl_enabled_for_all_reduce,
6
)
7
8
@@ -17,12 +15,8 @@ def tensor_model_parallel_all_reduce(input_):
15
if get_tensor_model_parallel_world_size() == 1:
16
return input_
17
# All-reduce.
20
- if is_custom_nccl_enabled_for_all_reduce():
21
- # TODO: support multiple parallel groups.
22
- cupy_utils.all_reduce(input_)
23
- else:
24
- torch.distributed.all_reduce(input_,
25
- group=get_tensor_model_parallel_group())
18
+ torch.distributed.all_reduce(input_,
19
+ group=get_tensor_model_parallel_group())
20
return input_
21
22
vllm/model_executor/parallel_utils/cupy_utils.py
deleted
-115
@@ -1,115 +0,0 @@
1
-"""CuPy utilities for all-reduce.
2
-
3
-We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4
-CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5
-CUDA graphs.
6
-
7
-TODO: Remove this file when torch.distributed.all_reduce is fixed.
8
-"""
9
-import contextlib
10
-
11
-import torch
12
-from torch.distributed import ReduceOp
13
-
14
-try:
15
- import cupy
16
- from cupyx.distributed import NCCLBackend
17
- from cupy.cuda import nccl
18
-except ImportError as e:
19
- cupy = e
20
- nccl = None
21
-
22
- class NCCLBackend:
23
- ...
24
-
25
-
26
-_OP_MAPPING = {
27
- ReduceOp.SUM: "sum",
28
- ReduceOp.PRODUCT: "prod",
29
- ReduceOp.MIN: "min",
30
- ReduceOp.MAX: "max",
31
-}
32
-
33
-
34
-class NCCLBackendWithBFloat16(NCCLBackend):
35
- # This is enough to add bfloat16 support for most operations,
36
- # but broadcast will fail (will require changes in compiled
37
- # cupy code).
38
- def _get_nccl_dtype_and_count(self, array, count=None):
39
- nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
40
- torch_dtype = getattr(array, "_torch_dtype", None)
41
- if torch_dtype is torch.bfloat16:
42
- nccl_dtype = nccl.NCCL_BFLOAT16
43
- return nccl_dtype, count
44
-
45
-
46
-_NCCL_BACKEND = None
47
-_WORLD_SIZE = 0
48
-
49
-
50
-def is_initialized() -> bool:
51
- """Returns whether the NCCL backend is initialized."""
52
- return _NCCL_BACKEND is not None
53
-
54
-
55
-@contextlib.contextmanager
56
-def set_cupy_stream(stream: torch.cuda.Stream) -> None:
57
- """Set the cuda stream for communication"""
58
- cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
59
- stream.device_index)
60
- with cupy_stream:
61
- yield
62
-
63
-
64
-def init_process_group(world_size: int, rank: int, host: str,
65
- port: int) -> None:
66
- """Initializes the CuPy NCCL backend.
67
-
68
- # TODO: handle NCCL timeouts.
69
- """
70
- assert not is_initialized()
71
-
72
- if isinstance(cupy, Exception):
73
- raise ImportError(
74
- "NCCLBackend is not available. Please install cupy.") from cupy
75
-
76
- # TODO(woosuk): Create TP and PP process groups for CuPy.
77
- global _NCCL_BACKEND
78
- global _WORLD_SIZE
79
- assert world_size > 0, f"{world_size=} should be a positive integer"
80
- assert 0 <= rank < world_size, (
81
- f"{rank=} should be a integer between [0, {world_size})")
82
-
83
- cupy.cuda.runtime.setDevice(torch.cuda.current_device())
84
- _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
85
- _WORLD_SIZE = world_size
86
-
87
-
88
-def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
89
- """All-reduces the input tensor across the process group."""
90
- assert input_.is_cuda, f"{input_} should be a cuda tensor"
91
- # Hack to support bfloat16
92
- torch_dtype = input_.dtype
93
- if torch_dtype is torch.bfloat16:
94
- # We need to view as float16, otherwise
95
- # cupy will fail. This will not change
96
- # the underlying data.
97
- input_ = input_.view(torch.float16)
98
- cupy_input = cupy.asarray(input_)
99
- cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
100
- _NCCL_BACKEND.all_reduce(in_array=cupy_input,
101
- out_array=cupy_input,
102
- op=_OP_MAPPING[op])
103
-
104
-
105
-def destroy_process_group() -> None:
106
- """Destroys the NCCL backend."""
107
- global _NCCL_BACKEND
108
- global _WORLD_SIZE
109
- _NCCL_BACKEND = None
110
- _WORLD_SIZE = 0
111
-
112
-
113
-def get_world_size() -> int:
114
- """Returns the world size."""
115
- return _WORLD_SIZE
vllm/model_executor/parallel_utils/parallel_state.py
-37
@@ -3,12 +3,9 @@
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
-import contextlib
6
7
import torch
8
10
-from vllm.model_executor.parallel_utils import cupy_utils
11
-
9
# Tensor model parallel group that the current rank belongs to.
10
_TENSOR_MODEL_PARALLEL_GROUP = None
11
# Pipeline model parallel group that the current rank belongs to.
@@ -180,37 +177,3 @@ def destroy_model_parallel():
177
_PIPELINE_MODEL_PARALLEL_GROUP = None
178
global _PIPELINE_GLOBAL_RANKS
179
_PIPELINE_GLOBAL_RANKS = None
183
-
184
- # Destroy the cupy states if any.
185
- cupy_utils.destroy_process_group()
186
-
187
-
188
-# Whether to use cupy for nccl all reduce.
189
-# We use cupy for all reduce when using CUDA graph, because torch.distributed
190
-# is not well supported by CUDA graph.
191
-_ENABLE_CUPY_FOR_ALL_REDUCE = False
192
-
193
-
194
-@contextlib.contextmanager
195
-def with_custom_nccl_for_all_reduce():
196
- """use custom nccl instead of torch.distributed for all reduce"""
197
- tp_size = get_tensor_model_parallel_world_size()
198
- if tp_size == 1:
199
- # No-op.
200
- # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
201
- yield
202
- else:
203
- global _ENABLE_CUPY_FOR_ALL_REDUCE
204
- old = _ENABLE_CUPY_FOR_ALL_REDUCE
205
- _ENABLE_CUPY_FOR_ALL_REDUCE = True
206
-
207
- stream = torch.cuda.current_stream()
208
- with cupy_utils.set_cupy_stream(stream):
209
- yield
210
- _ENABLE_CUPY_FOR_ALL_REDUCE = old
211
-
212
-
213
-def is_custom_nccl_enabled_for_all_reduce():
214
- """check if custom nccl is enabled for all reduce"""
215
- global _ENABLE_CUPY_FOR_ALL_REDUCE
216
- return _ENABLE_CUPY_FOR_ALL_REDUCE
vllm/worker/model_runner.py
+12
-18
@@ -8,8 +8,6 @@ import torch.nn as nn
8
from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9
from vllm.logger import init_logger
10
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11
-from vllm.model_executor.parallel_utils.parallel_state import (
12
- with_custom_nccl_for_all_reduce)
11
from vllm.sampling_params import SamplingParams, SamplingType
12
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
13
@@ -459,8 +457,18 @@ class CUDAGraphRunner:
457
# Run the model once without capturing the graph.
458
# This is to make sure that the captured graph does not include the
459
# kernel launches for initial benchmarking (e.g., Triton autotune).
462
- with with_custom_nccl_for_all_reduce():
463
- self.model(
460
+ self.model(
461
+ input_ids,
462
+ positions,
463
+ kv_caches,
464
+ input_metadata,
465
+ )
466
+ torch.cuda.synchronize()
467
+
468
+ # Capture the graph.
469
+ self.graph = torch.cuda.CUDAGraph()
470
+ with torch.cuda.graph(self.graph, pool=memory_pool):
471
+ hidden_states = self.model(
472
input_ids,
473
positions,
474
kv_caches,
@@ -468,20 +476,6 @@ class CUDAGraphRunner:
476
)
477
torch.cuda.synchronize()
478
471
- # Capture the graph.
472
- # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
473
- # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
474
- self.graph = torch.cuda.CUDAGraph()
475
- with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
476
- with with_custom_nccl_for_all_reduce():
477
- hidden_states = self.model(
478
- input_ids,
479
- positions,
480
- kv_caches,
481
- input_metadata,
482
- )
483
- torch.cuda.synchronize()
484
-
479
# Save the input and output buffers.
480
self.input_buffers = {
481
"input_ids": input_ids,
vllm/worker/worker.py
+4
-27
@@ -8,7 +8,6 @@ import torch.distributed
8
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
10
from vllm.model_executor import set_random_seed
11
-from vllm.model_executor.parallel_utils import cupy_utils
11
from vllm.model_executor.parallel_utils.parallel_state import (
12
initialize_model_parallel)
13
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -47,7 +46,7 @@ class Worker:
46
self.cache_events = None
47
self.gpu_cache = None
48
50
- def init_model(self, cupy_port: Optional[int] = None):
49
+ def init_model(self) -> None:
50
# torch.distributed.all_reduce does not free the input tensor until
51
# the synchronization point. This causes the memory usage to grow
52
# as the number of all_reduce calls increases. This env var disables
@@ -71,7 +70,7 @@ class Worker:
70
71
# Initialize the distributed environment.
72
_init_distributed_environment(self.parallel_config, self.rank,
74
- cupy_port, self.distributed_init_method)
73
+ self.distributed_init_method)
74
75
# Initialize the model.
76
set_random_seed(self.model_config.seed)
@@ -165,7 +164,6 @@ class Worker:
164
def _init_distributed_environment(
165
parallel_config: ParallelConfig,
166
rank: int,
168
- cupy_port: Optional[int],
167
distributed_init_method: Optional[str] = None,
168
) -> None:
169
"""Initialize the distributed environment."""
@@ -188,29 +186,8 @@ def _init_distributed_environment(
186
init_method=distributed_init_method,
187
)
188
191
- if cupy_utils.is_initialized():
192
- cupy_world_size = cupy_utils.get_world_size()
193
- if cupy_world_size != parallel_config.world_size:
194
- raise RuntimeError(
195
- "cupy.distributed is already initialized but the cupy world "
196
- "size does not match parallel_config.world_size "
197
- f"({cupy_world_size} vs. {parallel_config.world_size}).")
198
- elif parallel_config.world_size > 1:
199
- # NOTE(woosuk): We don't initialize CuPy process group when world size
200
- # is 1.
201
- # TODO(woosuk): Support multi-node connection.
202
- cupy_utils.init_process_group(
203
- world_size=parallel_config.world_size,
204
- rank=rank,
205
- host="localhost",
206
- port=cupy_port,
207
- )
208
-
209
- if parallel_config.world_size > 1:
210
- # A small all_reduce for warmup.
211
- torch.distributed.all_reduce(torch.zeros(1).cuda())
212
- cupy_utils.all_reduce(torch.zeros(1).cuda())
213
-
189
+ # A small all_reduce for warmup.
190
+ torch.distributed.all_reduce(torch.zeros(1).cuda())
191
initialize_model_parallel(parallel_config.tensor_parallel_size,
192
parallel_config.pipeline_parallel_size)
193