@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f44facddd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Core] remove cupy dependency (#3625)

youkaichao committed Mar 27, 2024 at 00:33 UTC 8f44facdddcf3c704f7d6a2719b6e85efc393449
17 files changed +506 -223
.buildkite/test-pipeline.yaml
+5 -2
@@ -22,10 +22,13 @@ steps:
22 working_dir: "/vllm-workspace/tests/distributed"
23 num_gpus: 2 # only support 1 or 2 for now.
24
25 -- label: Distributed Correctness Test
26 - command: pytest -v -s --forked test_basic_distributed_correctness.py
25 +- label: Distributed Tests
26 working_dir: "/vllm-workspace/tests/distributed"
27 num_gpus: 2 # only support 1 or 2 for now.
28 + commands:
29 + - pytest -v -s --forked test_pynccl.py
30 + - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s --forked test_basic_distributed_correctness.py
31 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s --forked test_basic_distributed_correctness.py
32
33 - label: Engine Test
34 command: pytest -v -s engine tokenization test_sequence.py test_config.py
Dockerfile
+1 -1
@@ -97,7 +97,7 @@ RUN --mount=type=cache,target=/root/.cache/pip VLLM_USE_PRECOMPILED=1 pip instal
97
98 #################### RUNTIME BASE IMAGE ####################
99 # We used base cuda image because pytorch installs its own cuda libraries.
100 -# However cupy depends on cuda libraries so we had to switch to the runtime image
100 +# However pynccl depends on cuda libraries so we had to switch to the runtime image
101 # In the future it would be nice to get a container with pytorch and cuda without duplicating cuda
102 FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS vllm-base
103
Dockerfile.rocm
-20
@@ -23,9 +23,6 @@ RUN echo "FA_BRANCH is $FA_BRANCH"
23 # In that case, we need to use the python reference attention implementation in vllm
24 ARG BUILD_FA="1"
25
26 -# whether to build cupy on rocm
27 -ARG BUILD_CUPY="1"
28 -
26 # Install some basic utilities
27 RUN apt-get update && apt-get install python3 python3-pip -y
28
@@ -78,23 +75,6 @@ RUN if [ "$BUILD_FA" = "1" ]; then \
75 RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
76 rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
77
81 -# build cupy
82 -RUN if [ "$BUILD_CUPY" = "1" ]; then \
83 - mkdir -p libs \
84 - && cd libs \
85 - && git clone -b hipgraph_enablement --recursive https://github.com/ROCm/cupy.git \
86 - && cd cupy \
87 - && pip install mpi4py-mpich \
88 - && pip install scipy==1.9.3 \
89 - && pip install cython==0.29.* \
90 - && env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py \
91 - && export CUPY_INSTALL_USE_HIP=1 \
92 - && export ROCM_HOME=/opt/rocm \
93 - && export HCC_AMDGPU_TARGET="gfx90a,gfx942,gfx1100" \
94 - && pip install . \
95 - && cd ..; \
96 - fi
97 -
78 COPY ./ /app/vllm
79
80 RUN python3 -m pip install --upgrade pip
requirements.txt
-1
@@ -14,4 +14,3 @@ prometheus_client >= 0.18.0
14 pynvml == 11.5.0
15 triton >= 2.1.0
16 outlines == 0.0.34
17 -cupy-cuda12x == 12.1.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead.
setup.py
-6
@@ -306,12 +306,6 @@ def get_requirements() -> List[str]:
306 if _is_cuda():
307 with open(get_path("requirements.txt")) as f:
308 requirements = f.read().strip().split("\n")
309 - if get_nvcc_cuda_version() <= Version("11.8"):
310 - # replace cupy-cuda12x with cupy-cuda11x for cuda 11.x
311 - for i in range(len(requirements)):
312 - if requirements[i].startswith("cupy-cuda12x"):
313 - requirements[i] = "cupy-cuda11x"
314 - break
309 elif _is_hip():
310 with open(get_path("requirements-rocm.txt")) as f:
311 requirements = f.read().strip().split("\n")
tests/distributed/test_basic_distributed_correctness.py
+13 -4
@@ -1,13 +1,22 @@
1 """Compare the outputs of HF and distributed vLLM when using greedy sampling.
2 -
3 -Run `pytest tests/distributed/test_basic_distributed_correctness.py --forked`.
2 +vLLM will allocate all the available memory, so we need to run the tests one
3 +by one. The solution is to pass arguments (model name) by environment
4 +variables.
5 +Run:
6 +```sh
7 +TEST_DIST_MODEL=facebook/opt-125m pytest \
8 + test_basic_distributed_correctness.py
9 +TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
10 + test_basic_distributed_correctness.py
11 +```
12 """
13 +import os
14 +
15 import pytest
16 import torch
17
18 MODELS = [
9 - "facebook/opt-125m",
10 - "meta-llama/Llama-2-7b-hf",
19 + os.environ["TEST_DIST_MODEL"],
20 ]
21
22
tests/distributed/test_comm_ops.py
+20
@@ -2,6 +2,8 @@
2
3 Run `pytest tests/distributed/test_comm_ops.py --forked`.
4 """
5 +import os
6 +
7 import pytest
8 import ray
9 import torch
@@ -16,6 +18,12 @@ from vllm.test_utils import (init_test_distributed_environment,
18 @ray.remote(num_gpus=1, max_calls=1)
19 def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
20 distributed_init_port: str):
21 + # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
22 + # so that each worker can see all the GPUs
23 + # they will be able to set the device to the correct GPU
24 + del os.environ["CUDA_VISIBLE_DEVICES"]
25 + device = torch.device(f"cuda:{rank}")
26 + torch.cuda.set_device(device)
27 init_test_distributed_environment(1, tensor_parallel_size, rank,
28 distributed_init_port)
29 num_elements = 8
@@ -32,6 +40,12 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
40 @ray.remote(num_gpus=1, max_calls=1)
41 def all_gather_test_worker(tensor_parallel_size: int, rank: int,
42 distributed_init_port: str):
43 + # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
44 + # so that each worker can see all the GPUs
45 + # they will be able to set the device to the correct GPU
46 + del os.environ["CUDA_VISIBLE_DEVICES"]
47 + device = torch.device(f"cuda:{rank}")
48 + torch.cuda.set_device(device)
49 init_test_distributed_environment(1, tensor_parallel_size, rank,
50 distributed_init_port)
51 num_dimensions = 3
@@ -54,6 +68,12 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
68 @ray.remote(num_gpus=1, max_calls=1)
69 def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
70 distributed_init_port: str):
71 + # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
72 + # so that each worker can see all the GPUs
73 + # they will be able to set the device to the correct GPU
74 + del os.environ["CUDA_VISIBLE_DEVICES"]
75 + device = torch.device(f"cuda:{rank}")
76 + torch.cuda.set_device(device)
77 init_test_distributed_environment(1, tensor_parallel_size, rank,
78 distributed_init_port)
79 test_dict = {
tests/distributed/test_pynccl.py new
+90
@@ -0,0 +1,90 @@
1 +import multiprocessing
2 +import os
3 +
4 +import pytest
5 +import torch
6 +
7 +from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
8 + ncclGetUniqueId)
9 +
10 +
11 +def distributed_run(fn, world_size):
12 + number_of_processes = world_size
13 + processes = []
14 + for i in range(number_of_processes):
15 + env = os.environ.copy()
16 + env['RANK'] = str(i)
17 + env['WORLD_SIZE'] = str(number_of_processes)
18 + env['MASTER_ADDR'] = 'localhost'
19 + env['MASTER_PORT'] = '12345'
20 + p = multiprocessing.Process(target=fn, args=(env, ))
21 + processes.append(p)
22 + p.start()
23 +
24 + for p in processes:
25 + p.join()
26 +
27 +
28 +def update_env(fn):
29 + # `multiprocessing.Process` cannot accept environment variables directly
30 + # so we need to pass the environment variables as arguments
31 + # and update the environment variables in the function
32 + def wrapper(env):
33 + import os
34 + os.environ.update(env)
35 + fn()
36 +
37 + return wrapper
38 +
39 +
40 +@update_env
41 +def worker_fn():
42 + comm = NCCLCommunicator()
43 + tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
44 + comm.all_reduce(tensor)
45 + result = tensor.mean().cpu().item()
46 + assert result == comm.world_size
47 +
48 +
49 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
50 + reason="Need at least 2 GPUs to run the test.")
51 +def test_pynccl():
52 + distributed_run(worker_fn, 2)
53 +
54 +
55 +@update_env
56 +def worker_fn_with_cudagraph():
57 + with torch.no_grad():
58 + graph = torch.cuda.CUDAGraph()
59 + comm = NCCLCommunicator()
60 + # run something in the default stream to initialize torch engine
61 + a = torch.ones((4, 4), device=f'cuda:{comm.rank}')
62 + torch.cuda.synchronize()
63 + with torch.cuda.graph(graph, stream=comm.stream):
64 + # operation during the graph capture is recorded but not executed
65 + # see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#creating-a-graph-using-stream-capture # noqa
66 + comm.all_reduce(a)
67 + comm.stream.synchronize()
68 + assert a.mean().cpu().item() == comm.world_size**0
69 + graph.replay()
70 + comm.stream.synchronize()
71 + assert a.mean().cpu().item() == comm.world_size**1
72 +
73 +
74 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
75 + reason="Need at least 2 GPUs to run the test.")
76 +def test_pynccl_with_cudagraph():
77 + distributed_run(worker_fn_with_cudagraph, 2)
78 +
79 +
80 +def test_ncclGetUniqueId():
81 + unique_id = ncclGetUniqueId()
82 + # `list(unique_id.internal)` is something like this:
83 + # [34, -16, 23, 83, 109, -19, 59, 95, 2, 0, -86, 55, 10, -128, 0, 29, 0,
84 + # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
85 + # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
86 + # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
87 + # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
88 + # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
89 + # as long as the function doesn't raise an exception, we're good
90 + assert unique_id is not None
vllm/executor/ray_gpu_executor.py
+2 -4
@@ -188,11 +188,9 @@ class RayGPUExecutor(ExecutorBase):
188 is_driver_worker=True,
189 )
190
191 - # FIXME(woosuk): We are not properly initializing cupy NCCL when
191 + # FIXME(woosuk): We are not properly initializing pynccl when
192 # we have multiple nodes.
193 - self._run_workers("init_device",
194 - cupy_port=get_open_port()
195 - if not model_config.enforce_eager else None)
193 + self._run_workers("init_device")
194 self._run_workers(
195 "load_model",
196 max_concurrent_workers=self.parallel_config.
vllm/model_executor/parallel_utils/communication_op.py
+4 -4
@@ -4,12 +4,12 @@ from typing import Any, Dict, List, Optional, Union
4 import torch
5 from torch.distributed import ProcessGroup
6
7 -from vllm.model_executor.parallel_utils import cupy_utils
7 +from vllm.model_executor.parallel_utils import pynccl_utils
8 from vllm.model_executor.parallel_utils.custom_all_reduce import (
9 custom_all_reduce)
10 from vllm.model_executor.parallel_utils.parallel_state import (
11 get_tensor_model_parallel_group, get_tensor_model_parallel_rank,
12 - get_tensor_model_parallel_world_size, is_cupy_nccl_enabled_for_all_reduce)
12 + get_tensor_model_parallel_world_size, is_pynccl_enabled_for_all_reduce)
13
14
15 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -30,9 +30,9 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
30 out = custom_all_reduce(input_)
31 if out is not None:
32 return out
33 - if is_cupy_nccl_enabled_for_all_reduce():
33 + if is_pynccl_enabled_for_all_reduce():
34 # TODO: support multiple parallel groups.
35 - cupy_utils.all_reduce(input_)
35 + pynccl_utils.all_reduce(input_)
36 else:
37 torch.distributed.all_reduce(input_,
38 group=get_tensor_model_parallel_group())
vllm/model_executor/parallel_utils/cupy_utils.py deleted
-130
@@ -1,130 +0,0 @@
1 -"""CuPy utilities for all-reduce.
2 -
3 -We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4 -CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5 -CUDA graphs.
6 -
7 -NOTE: We use CuPy 12.3 since CuPy 13.0 does not support Python 3.8.
8 -TODO: Remove this file when torch.distributed.all_reduce is fixed.
9 -"""
10 -import contextlib
11 -
12 -import torch
13 -from torch.distributed import ReduceOp
14 -
15 -try:
16 - import cupy
17 - from cupy.cuda import nccl
18 - from cupyx.distributed import NCCLBackend
19 -except ImportError as e:
20 - cupy = e
21 - nccl = None
22 -
23 - class NCCLBackend:
24 - ...
25 -
26 -
27 -_OP_MAPPING = {
28 - ReduceOp.SUM: "sum",
29 - ReduceOp.PRODUCT: "prod",
30 - ReduceOp.MIN: "min",
31 - ReduceOp.MAX: "max",
32 -}
33 -
34 -
35 -class NCCLBackendWithBFloat16(NCCLBackend):
36 - # This is enough to add bfloat16 support for most operations,
37 - # but broadcast will fail (will require changes in compiled
38 - # cupy code).
39 - def _get_nccl_dtype_and_count(self, array, count=None):
40 - nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
41 - torch_dtype = getattr(array, "_torch_dtype", None)
42 - if torch_dtype is torch.bfloat16:
43 - nccl_dtype = nccl.NCCL_BFLOAT16
44 - return nccl_dtype, count
45 -
46 - def barrier(self) -> None:
47 - raise RuntimeError(
48 - "Currently, CuPy NCCL barrier is not supported since the TCP "
49 - "store is immediately stopped after the initialization.")
50 -
51 -
52 -_NCCL_BACKEND = None
53 -_WORLD_SIZE = 0
54 -
55 -
56 -def is_initialized() -> bool:
57 - """Returns whether the NCCL backend is initialized."""
58 - return _NCCL_BACKEND is not None
59 -
60 -
61 -@contextlib.contextmanager
62 -def set_cupy_stream(stream: torch.cuda.Stream):
63 - """Set the cuda stream for communication"""
64 - cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
65 - stream.device_index)
66 - with cupy_stream:
67 - yield
68 -
69 -
70 -def init_process_group(world_size: int, rank: int, host: str,
71 - port: int) -> None:
72 - """Initializes the CuPy NCCL backend.
73 -
74 - # TODO: handle NCCL timeouts.
75 - """
76 - assert not is_initialized()
77 -
78 - if isinstance(cupy, Exception):
79 - raise ImportError(
80 - "NCCLBackend is not available. Please install cupy.") from cupy
81 -
82 - # TODO(woosuk): Create TP and PP process groups for CuPy.
83 - global _NCCL_BACKEND
84 - global _WORLD_SIZE
85 - assert world_size > 0, f"{world_size=} should be a positive integer"
86 - assert 0 <= rank < world_size, (
87 - f"{rank=} should be a integer between [0, {world_size})")
88 -
89 - cupy.cuda.runtime.setDevice(torch.cuda.current_device())
90 - _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
91 - _WORLD_SIZE = world_size
92 -
93 - # Stop the TCP store to prevent the deadlock issues at termination time.
94 - # FIXME(woosuk): This is hacky. Find a more robust solution.
95 - if rank == 0 and hasattr(_NCCL_BACKEND, "_store"):
96 - _NCCL_BACKEND._store.stop()
97 -
98 -
99 -def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
100 - """All-reduces the input tensor across the process group."""
101 - assert input_.is_cuda, f"{input_} should be a cuda tensor"
102 - # Hack to support bfloat16
103 - torch_dtype = input_.dtype
104 - if torch_dtype is torch.bfloat16:
105 - # We need to view as float16, otherwise
106 - # cupy will fail. This will not change
107 - # the underlying data.
108 - input_ = input_.view(torch.float16)
109 - cupy_input = cupy.asarray(input_)
110 - cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
111 - _NCCL_BACKEND.all_reduce(in_array=cupy_input,
112 - out_array=cupy_input,
113 - op=_OP_MAPPING[op])
114 -
115 -
116 -def destroy_process_group() -> None:
117 - """Destroys the NCCL backend."""
118 - global _NCCL_BACKEND
119 - global _WORLD_SIZE
120 - _NCCL_BACKEND = None
121 - _WORLD_SIZE = 0
122 -
123 -
124 -def get_world_size() -> int:
125 - """Returns the world size."""
126 - return _WORLD_SIZE
127 -
128 -
129 -def get_nccl_backend():
130 - return _NCCL_BACKEND
vllm/model_executor/parallel_utils/parallel_state.py
+18 -18
@@ -7,7 +7,7 @@ import contextlib
7
8 import torch
9
10 -from vllm.model_executor.parallel_utils import cupy_utils
10 +from vllm.model_executor.parallel_utils import pynccl_utils
11
12 # Tensor model parallel group that the current rank belongs to.
13 _TENSOR_MODEL_PARALLEL_GROUP = None
@@ -210,36 +210,36 @@ def destroy_model_parallel():
210 global _PIPELINE_GLOBAL_RANKS
211 _PIPELINE_GLOBAL_RANKS = None
212
213 - # Destroy the cupy states if any.
214 - cupy_utils.destroy_process_group()
213 + # Destroy the pynccl states if any.
214 + pynccl_utils.destroy_process_group()
215
216
217 -# Whether to use cupy for nccl all reduce.
218 -# We use cupy for all reduce when using CUDA graph, because torch.distributed
217 +# Whether to use pynccl for nccl all reduce.
218 +# We use pynccl for all reduce when using CUDA graph, because torch.distributed
219 # is not well supported by CUDA graph.
220 -_ENABLE_CUPY_FOR_ALL_REDUCE = False
220 +_ENABLE_PYNCCL_FOR_ALL_REDUCE = False
221
222
223 @contextlib.contextmanager
224 -def with_cupy_nccl_for_all_reduce():
225 - """use CuPy nccl instead of torch.distributed for all reduce"""
224 +def with_pynccl_for_all_reduce():
225 + """use pynccl instead of torch.distributed for all reduce"""
226 tp_size = get_tensor_model_parallel_world_size()
227 if tp_size == 1:
228 # No-op.
229 - # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
229 + # NOTE(woosuk): We don't initialize pynccl when tp_size is 1.
230 yield
231 else:
232 - global _ENABLE_CUPY_FOR_ALL_REDUCE
233 - old = _ENABLE_CUPY_FOR_ALL_REDUCE
234 - _ENABLE_CUPY_FOR_ALL_REDUCE = True
232 + global _ENABLE_PYNCCL_FOR_ALL_REDUCE
233 + old = _ENABLE_PYNCCL_FOR_ALL_REDUCE
234 + _ENABLE_PYNCCL_FOR_ALL_REDUCE = True
235
236 stream = torch.cuda.current_stream()
237 - with cupy_utils.set_cupy_stream(stream):
237 + with pynccl_utils.set_pynccl_stream(stream):
238 yield
239 - _ENABLE_CUPY_FOR_ALL_REDUCE = old
239 + _ENABLE_PYNCCL_FOR_ALL_REDUCE = old
240
241
242 -def is_cupy_nccl_enabled_for_all_reduce():
243 - """check if CuPy nccl is enabled for all reduce"""
244 - global _ENABLE_CUPY_FOR_ALL_REDUCE
245 - return _ENABLE_CUPY_FOR_ALL_REDUCE
242 +def is_pynccl_enabled_for_all_reduce():
243 + """check if pynccl is enabled for all reduce"""
244 + global _ENABLE_PYNCCL_FOR_ALL_REDUCE
245 + return _ENABLE_PYNCCL_FOR_ALL_REDUCE
vllm/model_executor/parallel_utils/pynccl.py new
+258
@@ -0,0 +1,258 @@
1 +# This file is a pure Python wrapper for the NCCL library.
2 +# The main purpose is to use NCCL combined with CUDA graph.
3 +# Before writing this script, we tried the following approach:
4 +# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself
5 +# often gets stuck when initializing the NCCL communicator.
6 +# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce`
7 +# contains many other potential cuda APIs, that are not allowed during
8 +# capturing the CUDA graph. For further details, please check
9 +# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ .
10 +#
11 +# Another rejected idea is to write a C/C++ binding for NCCL. It is usually
12 +# doable, but we often encounter issues related with nccl versions, and need
13 +# to switch between different versions of NCCL. See
14 +# https://github.com/NVIDIA/nccl/issues/1234 for more details.
15 +# A C/C++ binding is not flexible enough to handle this. It requires
16 +# recompilation of the code every time we want to switch between different
17 +# versions. This current implementation, with a **pure** Python wrapper, is
18 +# more flexible. We can easily switch between different versions of NCCL by
19 +# changing the environment variable `VLLM_NCCL_SO_PATH`, or the `so_file`
20 +# variable in the code.
21 +
22 +import ctypes
23 +import datetime
24 +import logging
25 +import os
26 +
27 +# ===================== import region =====================
28 +import torch
29 +import torch.distributed as dist
30 +from torch.distributed import ReduceOp
31 +
32 +logger = logging.getLogger(__name__)
33 +
34 +so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
35 +
36 +# manually load the nccl library
37 +if so_file:
38 + logger.info(
39 + f"Loading nccl from environment variable VLLM_NCCL_SO_PATH={so_file}")
40 +else:
41 + if torch.version.cuda is not None:
42 + so_file = "libnccl.so"
43 + elif torch.version.hip is not None:
44 + so_file = "librccl.so"
45 + else:
46 + raise ValueError("NCCL only supports CUDA and ROCm backends.")
47 + logger.debug(f"Loading nccl from library {so_file}")
48 +
49 +try:
50 + nccl = ctypes.CDLL(so_file)
51 +except Exception as e:
52 + logger.error(
53 + f"Failed to load NCCL library from {so_file} ."
54 + "It is expected if you are not running on NVIDIA/AMD GPUs."
55 + "Otherwise please set the environment variable VLLM_NCCL_SO_PATH"
56 + " to point to the correct nccl library path.")
57 + raise e
58 +
59 +# === export types and functions from nccl to Python ===
60 +# for the original nccl definition, please check
61 +# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in
62 +
63 +ncclResult_t = ctypes.c_int
64 +
65 +# equivalent to c declaration:
66 +# ncclResult_t ncclGetVersion(int *version);
67 +_c_ncclGetVersion = nccl.ncclGetVersion
68 +_c_ncclGetVersion.restype = ctypes.c_int
69 +_c_ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
70 +
71 +
72 +def ncclGetVersion() -> str:
73 + version = ctypes.c_int()
74 + result = _c_ncclGetVersion(ctypes.byref(version))
75 + assert result == 0
76 + # something like 21903 --> "2.19.3"
77 + version_str = str(version.value)
78 + major = version_str[0].lstrip("0")
79 + minor = version_str[1:3].lstrip("0")
80 + patch = version_str[3:].lstrip("0")
81 + return f"{major}.{minor}.{patch}"
82 +
83 +
84 +class NcclUniqueId(ctypes.Structure):
85 + _fields_ = [("internal", ctypes.c_byte * 128)]
86 +
87 +
88 +# equivalent to c declaration:
89 +# ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId);
90 +_c_ncclGetUniqueId = nccl.ncclGetUniqueId
91 +_c_ncclGetUniqueId.restype = ctypes.c_int
92 +_c_ncclGetUniqueId.argtypes = [ctypes.POINTER(NcclUniqueId)]
93 +
94 +
95 +def ncclGetUniqueId() -> NcclUniqueId:
96 + unique_id = NcclUniqueId()
97 + result = _c_ncclGetUniqueId(ctypes.byref(unique_id))
98 + assert result == 0
99 + return unique_id
100 +
101 +
102 +# equivalent to c declaration:
103 +# ncclResult_t ncclCommInitRank(
104 +# ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank);
105 +# note that ncclComm_t is a pointer type, so the first argument
106 +# is a pointer to a pointer
107 +_c_ncclCommInitRank = nccl.ncclCommInitRank
108 +_c_ncclCommInitRank.restype = ctypes.c_int
109 +_c_ncclCommInitRank.argtypes = [
110 + ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, NcclUniqueId, ctypes.c_int
111 +]
112 +
113 +
114 +# enums
115 +class ncclDataType_t(ctypes.c_int):
116 + ncclInt8 = 0
117 + ncclChar = 0
118 + ncclUint8 = 1
119 + ncclInt32 = 2
120 + ncclInt = 2
121 + ncclUint32 = 3
122 + ncclInt64 = 4
123 + ncclUint64 = 5
124 + ncclFloat16 = 6
125 + ncclHalf = 6
126 + ncclFloat32 = 7
127 + ncclFloat = 7
128 + ncclFloat64 = 8
129 + ncclDouble = 8
130 + ncclBfloat16 = 9
131 + ncclNumTypes = 10
132 +
133 + @classmethod
134 + def from_torch(cls, dtype: torch.dtype) -> 'ncclDataType_t':
135 + if dtype == torch.int8:
136 + return cls.ncclInt8
137 + if dtype == torch.uint8:
138 + return cls.ncclUint8
139 + if dtype == torch.int32:
140 + return cls.ncclInt32
141 + if dtype == torch.int64:
142 + return cls.ncclInt64
143 + if dtype == torch.float16:
144 + return cls.ncclFloat16
145 + if dtype == torch.float32:
146 + return cls.ncclFloat32
147 + if dtype == torch.float64:
148 + return cls.ncclFloat64
149 + if dtype == torch.bfloat16:
150 + return cls.ncclBfloat16
151 + raise ValueError(f"Unsupported dtype: {dtype}")
152 +
153 +
154 +class ncclRedOp_t(ctypes.c_int):
155 + ncclSum = 0
156 + ncclProd = 1
157 + ncclMax = 2
158 + ncclMin = 3
159 + ncclAvg = 4
160 + ncclNumOps = 5
161 +
162 + @classmethod
163 + def from_torch(cls, op: ReduceOp) -> 'ncclRedOp_t':
164 + if op == ReduceOp.SUM:
165 + return cls.ncclSum
166 + if op == ReduceOp.PRODUCT:
167 + return cls.ncclProd
168 + if op == ReduceOp.MAX:
169 + return cls.ncclMax
170 + if op == ReduceOp.MIN:
171 + return cls.ncclMin
172 + if op == ReduceOp.AVG:
173 + return cls.ncclAvg
174 + raise ValueError(f"Unsupported op: {op}")
175 +
176 +
177 +# equivalent to c declaration:
178 +# ncclResult_t ncclAllReduce(
179 +# const void* sendbuff, void* recvbuff, size_t count,
180 +# ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
181 +# udaStream_t stream);
182 +# note that cudaStream_t is a pointer type, so the last argument is a pointer
183 +_c_ncclAllReduce = nccl.ncclAllReduce
184 +_c_ncclAllReduce.restype = ctypes.c_int
185 +_c_ncclAllReduce.argtypes = [
186 + ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclDataType_t,
187 + ncclRedOp_t, ctypes.c_void_p, ctypes.c_void_p
188 +]
189 +
190 +# equivalent to c declaration:
191 +# ncclResult_t ncclCommDestroy(ncclComm_t comm);
192 +_c_ncclCommDestroy = nccl.ncclCommDestroy
193 +_c_ncclCommDestroy.restype = ctypes.c_int
194 +_c_ncclCommDestroy.argtypes = [ctypes.c_void_p]
195 +
196 +
197 +class NCCLCommunicator:
198 +
199 + def __init__(
200 + self,
201 + backend=None,
202 + init_method=None,
203 + timeout=datetime.timedelta(seconds=10),
204 + world_size: int = -1,
205 + rank: int = -1,
206 + store=None,
207 + group_name: str = "",
208 + pg_options=None,
209 + ):
210 + if not dist.is_initialized():
211 + backend = backend or "nccl"
212 + assert backend == 'nccl', (
213 + "only use nccl backend for starting the NCCL communicator")
214 + dist.init_process_group(backend=backend,
215 + init_method=init_method,
216 + timeout=timeout,
217 + world_size=world_size,
218 + rank=rank,
219 + store=store,
220 + group_name=group_name,
221 + pg_options=pg_options)
222 + self.world_size = dist.get_world_size()
223 + self.rank = dist.get_rank()
224 + torch.cuda.set_device(self.rank)
225 + if self.rank == 0:
226 + self.unique_id = ncclGetUniqueId()
227 + else:
228 + self.unique_id = NcclUniqueId()
229 + tensor = torch.ByteTensor(list(self.unique_id.internal)).cuda(
230 + self.rank)
231 + dist.broadcast(tensor, src=0)
232 + byte_list = tensor.cpu().tolist()
233 + self.unique_id = NcclUniqueId()
234 + for i, byte in enumerate(byte_list):
235 + self.unique_id.internal[i] = byte
236 + self.comm = ctypes.c_void_p()
237 + result = _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
238 + self.unique_id, self.rank)
239 + assert result == 0
240 + self.stream = torch.cuda.Stream(device=f"cuda:{self.rank}")
241 +
242 + def all_reduce(self,
243 + tensor: torch.Tensor,
244 + op: ReduceOp = ReduceOp.SUM,
245 + stream=None):
246 + if stream is None:
247 + stream = self.stream
248 + result = _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
249 + ctypes.c_void_p(tensor.data_ptr()),
250 + tensor.numel(),
251 + ncclDataType_t.from_torch(tensor.dtype),
252 + ncclRedOp_t.from_torch(op), self.comm,
253 + ctypes.c_void_p(stream.cuda_stream))
254 + assert result == 0
255 +
256 + def __del__(self):
257 + dist.destroy_process_group()
258 + _c_ncclCommDestroy(self.comm)
vllm/model_executor/parallel_utils/pynccl_utils.py new
+64
@@ -0,0 +1,64 @@
1 +import contextlib
2 +import logging
3 +from typing import Optional
4 +
5 +import torch
6 +from torch.distributed import ReduceOp
7 +
8 +logger = logging.getLogger(__name__)
9 +
10 +try:
11 + from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
12 + ncclGetVersion)
13 + logger.info(f"vLLM is using nccl=={ncclGetVersion()}")
14 +except Exception as e:
15 + # in non-NVIDIA environments, we can't import the nccl module
16 + # e.g. when running on machines with AMD GPUs
17 + logger.info(f"Failed to import NCCL library: {e}")
18 + logger.info("It is expected if you are not running on NVIDIA GPUs.")
19 + pass
20 +
21 +comm: Optional["NCCLCommunicator"] = None
22 +
23 +
24 +def is_initialized() -> bool:
25 + """Returns whether the NCCL backend is initialized."""
26 + return comm is not None
27 +
28 +
29 +@contextlib.contextmanager
30 +def set_pynccl_stream(stream: torch.cuda.Stream):
31 + """Set the cuda stream for communication"""
32 + try:
33 + comm.stream = stream
34 + yield
35 + finally:
36 + pass
37 +
38 +
39 +def init_process_group(world_size: int, rank: int, init_method: str) -> None:
40 + assert not is_initialized()
41 + global comm
42 + comm = NCCLCommunicator(init_method=init_method,
43 + world_size=world_size,
44 + rank=rank)
45 +
46 +
47 +def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
48 + """All-reduces the input tensor across the process group."""
49 + assert input_.is_cuda, f"{input_} should be a cuda tensor"
50 + comm.all_reduce(input_, op)
51 +
52 +
53 +def destroy_process_group() -> None:
54 + global comm
55 + comm = None
56 +
57 +
58 +def get_world_size() -> int:
59 + """Returns the world size."""
60 + return comm.world_size
61 +
62 +
63 +def get_nccl_backend():
64 + return comm
vllm/test_utils.py
+1 -4
@@ -16,10 +16,7 @@ def init_test_distributed_environment(
16 worker_use_ray=True)
17 distributed_init_method = f"tcp://localhost:{distributed_init_port}"
18 init_distributed_environment(
19 - parallel_config,
20 - rank,
21 - cupy_port=None,
22 - distributed_init_method=distributed_init_method)
19 + parallel_config, rank, distributed_init_method=distributed_init_method)
20
21
22 def multi_process_tensor_parallel(
vllm/worker/model_runner.py
+16 -13
@@ -15,11 +15,11 @@ from vllm.lora.request import LoRARequest
15 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
16 from vllm.model_executor import SamplingMetadata
17 from vllm.model_executor.model_loader import get_model
18 -from vllm.model_executor.parallel_utils import cupy_utils, custom_all_reduce
18 +from vllm.model_executor.parallel_utils import custom_all_reduce, pynccl_utils
19 from vllm.model_executor.parallel_utils.communication_op import (
20 broadcast_tensor_dict)
21 from vllm.model_executor.parallel_utils.parallel_state import (
22 - with_cupy_nccl_for_all_reduce)
22 + with_pynccl_for_all_reduce)
23 from vllm.sampling_params import SamplingParams, SamplingType
24 from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
25 SequenceGroupMetadata)
@@ -764,7 +764,7 @@ class ModelRunner:
764 """
765 # NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
766 # deleted before the CUDA graphs.
767 - self.cupy_nccl_backend = cupy_utils.get_nccl_backend()
767 + self.pynccl_backend = pynccl_utils.get_nccl_backend()
768
769 assert not self.model_config.enforce_eager
770 logger.info("Capturing the model for CUDA graphs. This may lead to "
@@ -794,11 +794,11 @@ class ModelRunner:
794 ]
795
796 # NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
797 - # kernel, CuPy NCCL, and PyTorch NCCL. When using CUDA graph, we use
798 - # either custom all-reduce kernel or CuPy NCCL. When not using CUDA
797 + # kernel, pynccl, and PyTorch NCCL. When using CUDA graph, we use
798 + # either custom all-reduce kernel or pynccl. When not using CUDA
799 # graph, we use either custom all-reduce kernel or PyTorch NCCL.
800 # We always prioritize using custom all-reduce kernel but fall back
801 - # to PyTorch or CuPy NCCL if it is disabled or not supported.
801 + # to PyTorch or pynccl if it is disabled or not supported.
802 with custom_all_reduce.capture():
803 # NOTE: Capturing the largest batch size first may help reduce the
804 # memory usage of CUDA graph.
@@ -846,12 +846,14 @@ class ModelRunner:
846 logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
847
848 def __del__(self) -> None:
849 - # Delete the CUDA graphs before deleting the CuPy NCCL communicator.
849 + # Delete the CUDA graphs before deleting the pynccl communicator.
850 # NOTE(woosuk): This is necessary because otherwise deadlocks can
851 # happen.
852 # FIXME(woosuk): This is a bit hacky. Find a more robust solution.
853 + # TODO(youkaichao): when we get enough user feedback that pynccl is
854 + # more stable than cupy, we can remove this, e.g. in v0.4.1.
855 self.graph_runners.clear()
854 - self.cupy_nccl_backend = None
856 + self.pynccl_backend = None
857
858 @property
859 def vocab_size(self) -> int:
@@ -879,7 +881,7 @@ class CUDAGraphRunner:
881 # Run the model once without capturing the graph.
882 # This is to make sure that the captured graph does not include the
883 # kernel launches for initial benchmarking (e.g., Triton autotune).
882 - with _maybe_cupy_nccl():
884 + with _maybe_pynccl():
885 self.model(
886 input_ids,
887 positions,
@@ -894,7 +896,7 @@ class CUDAGraphRunner:
896 # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
897 self.graph = torch.cuda.CUDAGraph()
898 with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
897 - with _maybe_cupy_nccl():
899 + with _maybe_pynccl():
900 hidden_states = self.model(
901 input_ids,
902 positions,
@@ -947,9 +949,10 @@ class CUDAGraphRunner:
949
950
951 @contextlib.contextmanager
950 -def _maybe_cupy_nccl():
951 - if cupy_utils.is_initialized() and not custom_all_reduce.is_initialized():
952 - with with_cupy_nccl_for_all_reduce():
952 +def _maybe_pynccl():
953 + if pynccl_utils.is_initialized(
954 + ) and not custom_all_reduce.is_initialized():
955 + with with_pynccl_for_all_reduce():
956 yield
957 else:
958 yield
vllm/worker/worker.py
+14 -16
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10 ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11 from vllm.lora.request import LoRARequest
12 from vllm.model_executor import set_random_seed
13 -from vllm.model_executor.parallel_utils import cupy_utils
13 +from vllm.model_executor.parallel_utils import pynccl_utils
14 from vllm.model_executor.parallel_utils.communication_op import (
15 broadcast_tensor_dict)
16 from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
@@ -75,7 +75,7 @@ class Worker:
75 self.cache_engine = None
76 self.gpu_cache = None
77
78 - def init_device(self, cupy_port: Optional[int] = None) -> None:
78 + def init_device(self) -> None:
79 if self.device_config.device.type == "cuda":
80 # torch.distributed.all_reduce does not free the input tensor until
81 # the synchronization point. This causes the memory usage to grow
@@ -98,7 +98,7 @@ class Worker:
98 f"Not support device type: {self.device_config.device}")
99 # Initialize the distributed environment.
100 init_distributed_environment(self.parallel_config, self.rank,
101 - cupy_port, self.distributed_init_method)
101 + self.distributed_init_method)
102 # Set random seed.
103 set_random_seed(self.model_config.seed)
104
@@ -250,7 +250,6 @@ class Worker:
250 def init_distributed_environment(
251 parallel_config: ParallelConfig,
252 rank: int,
253 - cupy_port: Optional[int],
253 distributed_init_method: Optional[str] = None,
254 ) -> None:
255 """Initialize the distributed environment."""
@@ -273,28 +272,27 @@ def init_distributed_environment(
272 init_method=distributed_init_method,
273 )
274
276 - if cupy_utils.is_initialized():
277 - cupy_world_size = cupy_utils.get_world_size()
278 - if cupy_world_size != parallel_config.world_size:
275 + if pynccl_utils.is_initialized():
276 + pynccl_world_size = pynccl_utils.get_world_size()
277 + if pynccl_world_size != parallel_config.world_size:
278 raise RuntimeError(
280 - "cupy.distributed is already initialized but the cupy world "
279 + "pynccl is already initialized but the pynccl world "
280 "size does not match parallel_config.world_size "
282 - f"({cupy_world_size} vs. {parallel_config.world_size}).")
283 - elif (parallel_config.world_size > 1 and cupy_port is not None):
284 - # NOTE(woosuk): We don't initialize CuPy process group when world size
281 + f"({pynccl_world_size} vs. {parallel_config.world_size}).")
282 + elif parallel_config.world_size > 1:
283 + # NOTE(woosuk): We don't initialize pynccl process group when world size
284 # is 1.
285 # TODO(woosuk): Support multi-node connection.
287 - cupy_utils.init_process_group(
286 + pynccl_utils.init_process_group(
287 world_size=parallel_config.world_size,
288 rank=rank,
290 - host="localhost",
291 - port=cupy_port,
289 + init_method=distributed_init_method,
290 )
291
292 # A small all_reduce for warmup.
293 torch.distributed.all_reduce(torch.zeros(1).cuda())
296 - if cupy_utils.is_initialized():
297 - cupy_utils.all_reduce(torch.zeros(1).cuda())
294 + if pynccl_utils.is_initialized():
295 + pynccl_utils.all_reduce(torch.zeros(1).cuda())
296 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
297 parallel_config.pipeline_parallel_size)
298