@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c3372e87be
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Remove dependency on CuPy (#2152)

Woosuk Kwon committed Dec 17, 2023 at 01:49 UTC c3372e87bed990510e4ae0b39f151a34dea24f8b
7 files changed +19 -208
requirements.txt
-1
@@ -12,4 +12,3 @@ fastapi
12 uvicorn[standard]
13 pydantic == 1.10.13 # Required for OpenAI server.
14 aioprometheus[starlette]
15 -cupy-cuda12x # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead. # FIXME: Fix this in setup.py.
vllm/engine/llm_engine.py
+1 -2
@@ -17,7 +17,7 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
17 SequenceOutput, SequenceStatus)
18 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
19 get_tokenizer)
20 -from vllm.utils import Counter, get_open_port
20 +from vllm.utils import Counter
21
22 if ray:
23 from ray.air.util.torch_dist import init_torch_dist_process_group
@@ -190,7 +190,6 @@ class LLMEngine:
190 ))
191 self._run_workers(
192 "init_model",
193 - cupy_port=get_open_port(),
193 get_all_outputs=True,
194 )
195 self._run_workers(
vllm/model_executor/parallel_utils/communication_op.py
+2 -8
@@ -1,10 +1,8 @@
1 import torch
2
3 -from vllm.model_executor.parallel_utils import cupy_utils
3 from vllm.model_executor.parallel_utils.parallel_state import (
4 get_tensor_model_parallel_world_size,
5 get_tensor_model_parallel_group,
7 - is_custom_nccl_enabled_for_all_reduce,
6 )
7
8
@@ -17,12 +15,8 @@ def tensor_model_parallel_all_reduce(input_):
15 if get_tensor_model_parallel_world_size() == 1:
16 return input_
17 # All-reduce.
20 - if is_custom_nccl_enabled_for_all_reduce():
21 - # TODO: support multiple parallel groups.
22 - cupy_utils.all_reduce(input_)
23 - else:
24 - torch.distributed.all_reduce(input_,
25 - group=get_tensor_model_parallel_group())
18 + torch.distributed.all_reduce(input_,
19 + group=get_tensor_model_parallel_group())
20 return input_
21
22
vllm/model_executor/parallel_utils/cupy_utils.py deleted
-115
@@ -1,115 +0,0 @@
1 -"""CuPy utilities for all-reduce.
2 -
3 -We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4 -CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5 -CUDA graphs.
6 -
7 -TODO: Remove this file when torch.distributed.all_reduce is fixed.
8 -"""
9 -import contextlib
10 -
11 -import torch
12 -from torch.distributed import ReduceOp
13 -
14 -try:
15 - import cupy
16 - from cupyx.distributed import NCCLBackend
17 - from cupy.cuda import nccl
18 -except ImportError as e:
19 - cupy = e
20 - nccl = None
21 -
22 - class NCCLBackend:
23 - ...
24 -
25 -
26 -_OP_MAPPING = {
27 - ReduceOp.SUM: "sum",
28 - ReduceOp.PRODUCT: "prod",
29 - ReduceOp.MIN: "min",
30 - ReduceOp.MAX: "max",
31 -}
32 -
33 -
34 -class NCCLBackendWithBFloat16(NCCLBackend):
35 - # This is enough to add bfloat16 support for most operations,
36 - # but broadcast will fail (will require changes in compiled
37 - # cupy code).
38 - def _get_nccl_dtype_and_count(self, array, count=None):
39 - nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
40 - torch_dtype = getattr(array, "_torch_dtype", None)
41 - if torch_dtype is torch.bfloat16:
42 - nccl_dtype = nccl.NCCL_BFLOAT16
43 - return nccl_dtype, count
44 -
45 -
46 -_NCCL_BACKEND = None
47 -_WORLD_SIZE = 0
48 -
49 -
50 -def is_initialized() -> bool:
51 - """Returns whether the NCCL backend is initialized."""
52 - return _NCCL_BACKEND is not None
53 -
54 -
55 -@contextlib.contextmanager
56 -def set_cupy_stream(stream: torch.cuda.Stream) -> None:
57 - """Set the cuda stream for communication"""
58 - cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
59 - stream.device_index)
60 - with cupy_stream:
61 - yield
62 -
63 -
64 -def init_process_group(world_size: int, rank: int, host: str,
65 - port: int) -> None:
66 - """Initializes the CuPy NCCL backend.
67 -
68 - # TODO: handle NCCL timeouts.
69 - """
70 - assert not is_initialized()
71 -
72 - if isinstance(cupy, Exception):
73 - raise ImportError(
74 - "NCCLBackend is not available. Please install cupy.") from cupy
75 -
76 - # TODO(woosuk): Create TP and PP process groups for CuPy.
77 - global _NCCL_BACKEND
78 - global _WORLD_SIZE
79 - assert world_size > 0, f"{world_size=} should be a positive integer"
80 - assert 0 <= rank < world_size, (
81 - f"{rank=} should be a integer between [0, {world_size})")
82 -
83 - cupy.cuda.runtime.setDevice(torch.cuda.current_device())
84 - _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
85 - _WORLD_SIZE = world_size
86 -
87 -
88 -def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
89 - """All-reduces the input tensor across the process group."""
90 - assert input_.is_cuda, f"{input_} should be a cuda tensor"
91 - # Hack to support bfloat16
92 - torch_dtype = input_.dtype
93 - if torch_dtype is torch.bfloat16:
94 - # We need to view as float16, otherwise
95 - # cupy will fail. This will not change
96 - # the underlying data.
97 - input_ = input_.view(torch.float16)
98 - cupy_input = cupy.asarray(input_)
99 - cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
100 - _NCCL_BACKEND.all_reduce(in_array=cupy_input,
101 - out_array=cupy_input,
102 - op=_OP_MAPPING[op])
103 -
104 -
105 -def destroy_process_group() -> None:
106 - """Destroys the NCCL backend."""
107 - global _NCCL_BACKEND
108 - global _WORLD_SIZE
109 - _NCCL_BACKEND = None
110 - _WORLD_SIZE = 0
111 -
112 -
113 -def get_world_size() -> int:
114 - """Returns the world size."""
115 - return _WORLD_SIZE
vllm/model_executor/parallel_utils/parallel_state.py
-37
@@ -3,12 +3,9 @@
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 """Tensor and pipeline parallel groups."""
6 -import contextlib
6
7 import torch
8
10 -from vllm.model_executor.parallel_utils import cupy_utils
11 -
9 # Tensor model parallel group that the current rank belongs to.
10 _TENSOR_MODEL_PARALLEL_GROUP = None
11 # Pipeline model parallel group that the current rank belongs to.
@@ -180,37 +177,3 @@ def destroy_model_parallel():
177 _PIPELINE_MODEL_PARALLEL_GROUP = None
178 global _PIPELINE_GLOBAL_RANKS
179 _PIPELINE_GLOBAL_RANKS = None
183 -
184 - # Destroy the cupy states if any.
185 - cupy_utils.destroy_process_group()
186 -
187 -
188 -# Whether to use cupy for nccl all reduce.
189 -# We use cupy for all reduce when using CUDA graph, because torch.distributed
190 -# is not well supported by CUDA graph.
191 -_ENABLE_CUPY_FOR_ALL_REDUCE = False
192 -
193 -
194 -@contextlib.contextmanager
195 -def with_custom_nccl_for_all_reduce():
196 - """use custom nccl instead of torch.distributed for all reduce"""
197 - tp_size = get_tensor_model_parallel_world_size()
198 - if tp_size == 1:
199 - # No-op.
200 - # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
201 - yield
202 - else:
203 - global _ENABLE_CUPY_FOR_ALL_REDUCE
204 - old = _ENABLE_CUPY_FOR_ALL_REDUCE
205 - _ENABLE_CUPY_FOR_ALL_REDUCE = True
206 -
207 - stream = torch.cuda.current_stream()
208 - with cupy_utils.set_cupy_stream(stream):
209 - yield
210 - _ENABLE_CUPY_FOR_ALL_REDUCE = old
211 -
212 -
213 -def is_custom_nccl_enabled_for_all_reduce():
214 - """check if custom nccl is enabled for all reduce"""
215 - global _ENABLE_CUPY_FOR_ALL_REDUCE
216 - return _ENABLE_CUPY_FOR_ALL_REDUCE
vllm/worker/model_runner.py
+12 -18
@@ -8,8 +8,6 @@ import torch.nn as nn
8 from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9 from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 -from vllm.model_executor.parallel_utils.parallel_state import (
12 - with_custom_nccl_for_all_reduce)
11 from vllm.sampling_params import SamplingParams, SamplingType
12 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
13
@@ -459,8 +457,18 @@ class CUDAGraphRunner:
457 # Run the model once without capturing the graph.
458 # This is to make sure that the captured graph does not include the
459 # kernel launches for initial benchmarking (e.g., Triton autotune).
462 - with with_custom_nccl_for_all_reduce():
463 - self.model(
460 + self.model(
461 + input_ids,
462 + positions,
463 + kv_caches,
464 + input_metadata,
465 + )
466 + torch.cuda.synchronize()
467 +
468 + # Capture the graph.
469 + self.graph = torch.cuda.CUDAGraph()
470 + with torch.cuda.graph(self.graph, pool=memory_pool):
471 + hidden_states = self.model(
472 input_ids,
473 positions,
474 kv_caches,
@@ -468,20 +476,6 @@ class CUDAGraphRunner:
476 )
477 torch.cuda.synchronize()
478
471 - # Capture the graph.
472 - # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
473 - # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
474 - self.graph = torch.cuda.CUDAGraph()
475 - with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
476 - with with_custom_nccl_for_all_reduce():
477 - hidden_states = self.model(
478 - input_ids,
479 - positions,
480 - kv_caches,
481 - input_metadata,
482 - )
483 - torch.cuda.synchronize()
484 -
479 # Save the input and output buffers.
480 self.input_buffers = {
481 "input_ids": input_ids,
vllm/worker/worker.py
+4 -27
@@ -8,7 +8,6 @@ import torch.distributed
8 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
10 from vllm.model_executor import set_random_seed
11 -from vllm.model_executor.parallel_utils import cupy_utils
11 from vllm.model_executor.parallel_utils.parallel_state import (
12 initialize_model_parallel)
13 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -47,7 +46,7 @@ class Worker:
46 self.cache_events = None
47 self.gpu_cache = None
48
50 - def init_model(self, cupy_port: Optional[int] = None):
49 + def init_model(self) -> None:
50 # torch.distributed.all_reduce does not free the input tensor until
51 # the synchronization point. This causes the memory usage to grow
52 # as the number of all_reduce calls increases. This env var disables
@@ -71,7 +70,7 @@ class Worker:
70
71 # Initialize the distributed environment.
72 _init_distributed_environment(self.parallel_config, self.rank,
74 - cupy_port, self.distributed_init_method)
73 + self.distributed_init_method)
74
75 # Initialize the model.
76 set_random_seed(self.model_config.seed)
@@ -165,7 +164,6 @@ class Worker:
164 def _init_distributed_environment(
165 parallel_config: ParallelConfig,
166 rank: int,
168 - cupy_port: Optional[int],
167 distributed_init_method: Optional[str] = None,
168 ) -> None:
169 """Initialize the distributed environment."""
@@ -188,29 +186,8 @@ def _init_distributed_environment(
186 init_method=distributed_init_method,
187 )
188
191 - if cupy_utils.is_initialized():
192 - cupy_world_size = cupy_utils.get_world_size()
193 - if cupy_world_size != parallel_config.world_size:
194 - raise RuntimeError(
195 - "cupy.distributed is already initialized but the cupy world "
196 - "size does not match parallel_config.world_size "
197 - f"({cupy_world_size} vs. {parallel_config.world_size}).")
198 - elif parallel_config.world_size > 1:
199 - # NOTE(woosuk): We don't initialize CuPy process group when world size
200 - # is 1.
201 - # TODO(woosuk): Support multi-node connection.
202 - cupy_utils.init_process_group(
203 - world_size=parallel_config.world_size,
204 - rank=rank,
205 - host="localhost",
206 - port=cupy_port,
207 - )
208 -
209 - if parallel_config.world_size > 1:
210 - # A small all_reduce for warmup.
211 - torch.distributed.all_reduce(torch.zeros(1).cuda())
212 - cupy_utils.all_reduce(torch.zeros(1).cuda())
213 -
189 + # A small all_reduce for warmup.
190 + torch.distributed.all_reduce(torch.zeros(1).cuda())
191 initialize_model_parallel(parallel_config.tensor_parallel_size,
192 parallel_config.pipeline_parallel_size)
193