A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Don't use cupy NCCL for AMD backends (#2855)
Woosuk Kwon committed
Feb 14, 2024 at 12:30 UTC
25e86b6a616638cea9ce121a6c28c7b1d69615e7
3 files changed
+23
-7
vllm/model_executor/parallel_utils/custom_all_reduce.py
+4
@@ -67,6 +67,10 @@ def get_handle() -> Optional["CustomAllreduce"]:
67
return _CA_HANDLE
68
69
70
+def is_initialized() -> bool:
71
+ return _CA_HANDLE is not None
72
+
73
+
74
@contextmanager
75
def capture():
76
try:
vllm/worker/model_runner.py
+16
-6
@@ -1,3 +1,4 @@
1
+import contextlib
2
import time
3
from typing import Dict, List, Optional, Tuple, Set, Union
4
@@ -9,9 +10,9 @@ from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
10
SchedulerConfig)
11
from vllm.logger import init_logger
12
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
13
+from vllm.model_executor.parallel_utils import cupy_utils
14
from vllm.model_executor.parallel_utils.communication_op import (
15
broadcast_tensor_dict)
14
-from vllm.model_executor.parallel_utils.cupy_utils import get_nccl_backend
16
from vllm.model_executor.parallel_utils.parallel_state import (
17
with_cupy_nccl_for_all_reduce)
18
from vllm.model_executor.parallel_utils import custom_all_reduce
@@ -659,7 +660,7 @@ class ModelRunner:
660
def capture_model(self, kv_caches: List[KVCache]) -> None:
661
# NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
662
# deleted before the CUDA graphs.
662
- self.cupy_nccl_backend = get_nccl_backend()
663
+ self.cupy_nccl_backend = cupy_utils.get_nccl_backend()
664
665
assert not self.model_config.enforce_eager
666
logger.info("Capturing the model for CUDA graphs. This may lead to "
@@ -689,8 +690,6 @@ class ModelRunner:
690
bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
691
]
692
692
- # NOTE: Capturing the largest batch size first may help reduce the
693
- # memory usage of CUDA graph.
693
# NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
694
# kernel, CuPy NCCL, and PyTorch NCCL. When using CUDA graph, we use
695
# either custom all-reduce kernel or CuPy NCCL. When not using CUDA
@@ -698,6 +697,8 @@ class ModelRunner:
697
# We always prioritize using custom all-reduce kernel but fall back
698
# to PyTorch or CuPy NCCL if it is disabled or not supported.
699
with custom_all_reduce.capture():
700
+ # NOTE: Capturing the largest batch size first may help reduce the
701
+ # memory usage of CUDA graph.
702
for batch_size in reversed(batch_size_capture_list):
703
# Create dummy input_metadata.
704
input_metadata = InputMetadata(
@@ -765,7 +766,7 @@ class CUDAGraphRunner:
766
# Run the model once without capturing the graph.
767
# This is to make sure that the captured graph does not include the
768
# kernel launches for initial benchmarking (e.g., Triton autotune).
768
- with with_cupy_nccl_for_all_reduce():
769
+ with _maybe_cupy_nccl():
770
self.model(
771
input_ids,
772
positions,
@@ -779,7 +780,7 @@ class CUDAGraphRunner:
780
# https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
781
self.graph = torch.cuda.CUDAGraph()
782
with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
782
- with with_cupy_nccl_for_all_reduce():
783
+ with _maybe_cupy_nccl():
784
hidden_states = self.model(
785
input_ids,
786
positions,
@@ -830,6 +831,15 @@ class CUDAGraphRunner:
831
return self.forward(*args, **kwargs)
832
833
834
+@contextlib.contextmanager
835
+def _maybe_cupy_nccl():
836
+ if cupy_utils.is_initialized() and not custom_all_reduce.is_initialized():
837
+ with with_cupy_nccl_for_all_reduce():
838
+ yield
839
+ else:
840
+ yield
841
+
842
+
843
def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
844
assert len(x) <= max_len
845
return x + [pad] * (max_len - len(x))
vllm/worker/worker.py
+3
-1
@@ -19,6 +19,7 @@ from vllm.sequence import SamplerOutput, SequenceGroupMetadata
19
from vllm.worker.cache_engine import CacheEngine
20
from vllm.worker.model_runner import ModelRunner
21
from vllm.lora.request import LoRARequest
22
+from vllm.utils import is_hip
23
24
25
class Worker:
@@ -268,7 +269,8 @@ def init_distributed_environment(
269
"cupy.distributed is already initialized but the cupy world "
270
"size does not match parallel_config.world_size "
271
f"({cupy_world_size} vs. {parallel_config.world_size}).")
271
- elif parallel_config.world_size > 1 and cupy_port is not None:
272
+ elif (parallel_config.world_size > 1 and cupy_port is not None
273
+ and not is_hip()):
274
# NOTE(woosuk): We don't initialize CuPy process group when world size
275
# is 1.
276
# TODO(woosuk): Support multi-node connection.