@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 25e86b6a61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Don't use cupy NCCL for AMD backends (#2855)

Woosuk Kwon committed Feb 14, 2024 at 12:30 UTC 25e86b6a616638cea9ce121a6c28c7b1d69615e7
3 files changed +23 -7
vllm/model_executor/parallel_utils/custom_all_reduce.py
+4
@@ -67,6 +67,10 @@ def get_handle() -> Optional["CustomAllreduce"]:
67 return _CA_HANDLE
68
69
70 +def is_initialized() -> bool:
71 + return _CA_HANDLE is not None
72 +
73 +
74 @contextmanager
75 def capture():
76 try:
vllm/worker/model_runner.py
+16 -6
@@ -1,3 +1,4 @@
1 +import contextlib
2 import time
3 from typing import Dict, List, Optional, Tuple, Set, Union
4
@@ -9,9 +10,9 @@ from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
10 SchedulerConfig)
11 from vllm.logger import init_logger
12 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
13 +from vllm.model_executor.parallel_utils import cupy_utils
14 from vllm.model_executor.parallel_utils.communication_op import (
15 broadcast_tensor_dict)
14 -from vllm.model_executor.parallel_utils.cupy_utils import get_nccl_backend
16 from vllm.model_executor.parallel_utils.parallel_state import (
17 with_cupy_nccl_for_all_reduce)
18 from vllm.model_executor.parallel_utils import custom_all_reduce
@@ -659,7 +660,7 @@ class ModelRunner:
660 def capture_model(self, kv_caches: List[KVCache]) -> None:
661 # NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
662 # deleted before the CUDA graphs.
662 - self.cupy_nccl_backend = get_nccl_backend()
663 + self.cupy_nccl_backend = cupy_utils.get_nccl_backend()
664
665 assert not self.model_config.enforce_eager
666 logger.info("Capturing the model for CUDA graphs. This may lead to "
@@ -689,8 +690,6 @@ class ModelRunner:
690 bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
691 ]
692
692 - # NOTE: Capturing the largest batch size first may help reduce the
693 - # memory usage of CUDA graph.
693 # NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
694 # kernel, CuPy NCCL, and PyTorch NCCL. When using CUDA graph, we use
695 # either custom all-reduce kernel or CuPy NCCL. When not using CUDA
@@ -698,6 +697,8 @@ class ModelRunner:
697 # We always prioritize using custom all-reduce kernel but fall back
698 # to PyTorch or CuPy NCCL if it is disabled or not supported.
699 with custom_all_reduce.capture():
700 + # NOTE: Capturing the largest batch size first may help reduce the
701 + # memory usage of CUDA graph.
702 for batch_size in reversed(batch_size_capture_list):
703 # Create dummy input_metadata.
704 input_metadata = InputMetadata(
@@ -765,7 +766,7 @@ class CUDAGraphRunner:
766 # Run the model once without capturing the graph.
767 # This is to make sure that the captured graph does not include the
768 # kernel launches for initial benchmarking (e.g., Triton autotune).
768 - with with_cupy_nccl_for_all_reduce():
769 + with _maybe_cupy_nccl():
770 self.model(
771 input_ids,
772 positions,
@@ -779,7 +780,7 @@ class CUDAGraphRunner:
780 # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
781 self.graph = torch.cuda.CUDAGraph()
782 with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
782 - with with_cupy_nccl_for_all_reduce():
783 + with _maybe_cupy_nccl():
784 hidden_states = self.model(
785 input_ids,
786 positions,
@@ -830,6 +831,15 @@ class CUDAGraphRunner:
831 return self.forward(*args, **kwargs)
832
833
834 +@contextlib.contextmanager
835 +def _maybe_cupy_nccl():
836 + if cupy_utils.is_initialized() and not custom_all_reduce.is_initialized():
837 + with with_cupy_nccl_for_all_reduce():
838 + yield
839 + else:
840 + yield
841 +
842 +
843 def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
844 assert len(x) <= max_len
845 return x + [pad] * (max_len - len(x))
vllm/worker/worker.py
+3 -1
@@ -19,6 +19,7 @@ from vllm.sequence import SamplerOutput, SequenceGroupMetadata
19 from vllm.worker.cache_engine import CacheEngine
20 from vllm.worker.model_runner import ModelRunner
21 from vllm.lora.request import LoRARequest
22 +from vllm.utils import is_hip
23
24
25 class Worker:
@@ -268,7 +269,8 @@ def init_distributed_environment(
269 "cupy.distributed is already initialized but the cupy world "
270 "size does not match parallel_config.world_size "
271 f"({cupy_world_size} vs. {parallel_config.world_size}).")
271 - elif parallel_config.world_size > 1 and cupy_port is not None:
272 + elif (parallel_config.world_size > 1 and cupy_port is not None
273 + and not is_hip()):
274 # NOTE(woosuk): We don't initialize CuPy process group when world size
275 # is 1.
276 # TODO(woosuk): Support multi-node connection.