A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Core] remove cupy dependency (#3625)
youkaichao committed
Mar 27, 2024 at 00:33 UTC
8f44facdddcf3c704f7d6a2719b6e85efc393449
17 files changed
+506
-223
.buildkite/test-pipeline.yaml
+5
-2
@@ -22,10 +22,13 @@ steps:
22
working_dir: "/vllm-workspace/tests/distributed"
23
num_gpus: 2 # only support 1 or 2 for now.
24
25
-- label: Distributed Correctness Test
26
- command: pytest -v -s --forked test_basic_distributed_correctness.py
25
+- label: Distributed Tests
26
working_dir: "/vllm-workspace/tests/distributed"
27
num_gpus: 2 # only support 1 or 2 for now.
28
+ commands:
29
+ - pytest -v -s --forked test_pynccl.py
30
+ - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s --forked test_basic_distributed_correctness.py
31
+ - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s --forked test_basic_distributed_correctness.py
32
33
- label: Engine Test
34
command: pytest -v -s engine tokenization test_sequence.py test_config.py
Dockerfile
+1
-1
@@ -97,7 +97,7 @@ RUN --mount=type=cache,target=/root/.cache/pip VLLM_USE_PRECOMPILED=1 pip instal
97
98
#################### RUNTIME BASE IMAGE ####################
99
# We used base cuda image because pytorch installs its own cuda libraries.
100
-# However cupy depends on cuda libraries so we had to switch to the runtime image
100
+# However pynccl depends on cuda libraries so we had to switch to the runtime image
101
# In the future it would be nice to get a container with pytorch and cuda without duplicating cuda
102
FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS vllm-base
103
Dockerfile.rocm
-20
@@ -23,9 +23,6 @@ RUN echo "FA_BRANCH is $FA_BRANCH"
23
# In that case, we need to use the python reference attention implementation in vllm
24
ARG BUILD_FA="1"
25
26
-# whether to build cupy on rocm
27
-ARG BUILD_CUPY="1"
28
-
26
# Install some basic utilities
27
RUN apt-get update && apt-get install python3 python3-pip -y
28
@@ -78,23 +75,6 @@ RUN if [ "$BUILD_FA" = "1" ]; then \
75
RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
76
rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
77
81
-# build cupy
82
-RUN if [ "$BUILD_CUPY" = "1" ]; then \
83
- mkdir -p libs \
84
- && cd libs \
85
- && git clone -b hipgraph_enablement --recursive https://github.com/ROCm/cupy.git \
86
- && cd cupy \
87
- && pip install mpi4py-mpich \
88
- && pip install scipy==1.9.3 \
89
- && pip install cython==0.29.* \
90
- && env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py \
91
- && export CUPY_INSTALL_USE_HIP=1 \
92
- && export ROCM_HOME=/opt/rocm \
93
- && export HCC_AMDGPU_TARGET="gfx90a,gfx942,gfx1100" \
94
- && pip install . \
95
- && cd ..; \
96
- fi
97
-
78
COPY ./ /app/vllm
79
80
RUN python3 -m pip install --upgrade pip
requirements.txt
-1
@@ -14,4 +14,3 @@ prometheus_client >= 0.18.0
14
pynvml == 11.5.0
15
triton >= 2.1.0
16
outlines == 0.0.34
17
-cupy-cuda12x == 12.1.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead.
setup.py
-6
@@ -306,12 +306,6 @@ def get_requirements() -> List[str]:
306
if _is_cuda():
307
with open(get_path("requirements.txt")) as f:
308
requirements = f.read().strip().split("\n")
309
- if get_nvcc_cuda_version() <= Version("11.8"):
310
- # replace cupy-cuda12x with cupy-cuda11x for cuda 11.x
311
- for i in range(len(requirements)):
312
- if requirements[i].startswith("cupy-cuda12x"):
313
- requirements[i] = "cupy-cuda11x"
314
- break
309
elif _is_hip():
310
with open(get_path("requirements-rocm.txt")) as f:
311
requirements = f.read().strip().split("\n")
tests/distributed/test_basic_distributed_correctness.py
+13
-4
@@ -1,13 +1,22 @@
1
"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
-
3
-Run `pytest tests/distributed/test_basic_distributed_correctness.py --forked`.
2
+vLLM will allocate all the available memory, so we need to run the tests one
3
+by one. The solution is to pass arguments (model name) by environment
4
+variables.
5
+Run:
6
+```sh
7
+TEST_DIST_MODEL=facebook/opt-125m pytest \
8
+ test_basic_distributed_correctness.py
9
+TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
10
+ test_basic_distributed_correctness.py
11
+```
12
"""
13
+import os
14
+
15
import pytest
16
import torch
17
18
MODELS = [
9
- "facebook/opt-125m",
10
- "meta-llama/Llama-2-7b-hf",
19
+ os.environ["TEST_DIST_MODEL"],
20
]
21
22
tests/distributed/test_comm_ops.py
+20
@@ -2,6 +2,8 @@
2
3
Run `pytest tests/distributed/test_comm_ops.py --forked`.
4
"""
5
+import os
6
+
7
import pytest
8
import ray
9
import torch
@@ -16,6 +18,12 @@ from vllm.test_utils import (init_test_distributed_environment,
18
@ray.remote(num_gpus=1, max_calls=1)
19
def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
20
distributed_init_port: str):
21
+ # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
22
+ # so that each worker can see all the GPUs
23
+ # they will be able to set the device to the correct GPU
24
+ del os.environ["CUDA_VISIBLE_DEVICES"]
25
+ device = torch.device(f"cuda:{rank}")
26
+ torch.cuda.set_device(device)
27
init_test_distributed_environment(1, tensor_parallel_size, rank,
28
distributed_init_port)
29
num_elements = 8
@@ -32,6 +40,12 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
40
@ray.remote(num_gpus=1, max_calls=1)
41
def all_gather_test_worker(tensor_parallel_size: int, rank: int,
42
distributed_init_port: str):
43
+ # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
44
+ # so that each worker can see all the GPUs
45
+ # they will be able to set the device to the correct GPU
46
+ del os.environ["CUDA_VISIBLE_DEVICES"]
47
+ device = torch.device(f"cuda:{rank}")
48
+ torch.cuda.set_device(device)
49
init_test_distributed_environment(1, tensor_parallel_size, rank,
50
distributed_init_port)
51
num_dimensions = 3
@@ -54,6 +68,12 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
68
@ray.remote(num_gpus=1, max_calls=1)
69
def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
70
distributed_init_port: str):
71
+ # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
72
+ # so that each worker can see all the GPUs
73
+ # they will be able to set the device to the correct GPU
74
+ del os.environ["CUDA_VISIBLE_DEVICES"]
75
+ device = torch.device(f"cuda:{rank}")
76
+ torch.cuda.set_device(device)
77
init_test_distributed_environment(1, tensor_parallel_size, rank,
78
distributed_init_port)
79
test_dict = {
tests/distributed/test_pynccl.py
new
+90
@@ -0,0 +1,90 @@
1
+import multiprocessing
2
+import os
3
+
4
+import pytest
5
+import torch
6
+
7
+from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
8
+ ncclGetUniqueId)
9
+
10
+
11
+def distributed_run(fn, world_size):
12
+ number_of_processes = world_size
13
+ processes = []
14
+ for i in range(number_of_processes):
15
+ env = os.environ.copy()
16
+ env['RANK'] = str(i)
17
+ env['WORLD_SIZE'] = str(number_of_processes)
18
+ env['MASTER_ADDR'] = 'localhost'
19
+ env['MASTER_PORT'] = '12345'
20
+ p = multiprocessing.Process(target=fn, args=(env, ))
21
+ processes.append(p)
22
+ p.start()
23
+
24
+ for p in processes:
25
+ p.join()
26
+
27
+
28
+def update_env(fn):
29
+ # `multiprocessing.Process` cannot accept environment variables directly
30
+ # so we need to pass the environment variables as arguments
31
+ # and update the environment variables in the function
32
+ def wrapper(env):
33
+ import os
34
+ os.environ.update(env)
35
+ fn()
36
+
37
+ return wrapper
38
+
39
+
40
+@update_env
41
+def worker_fn():
42
+ comm = NCCLCommunicator()
43
+ tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
44
+ comm.all_reduce(tensor)
45
+ result = tensor.mean().cpu().item()
46
+ assert result == comm.world_size
47
+
48
+
49
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
50
+ reason="Need at least 2 GPUs to run the test.")
51
+def test_pynccl():
52
+ distributed_run(worker_fn, 2)
53
+
54
+
55
+@update_env
56
+def worker_fn_with_cudagraph():
57
+ with torch.no_grad():
58
+ graph = torch.cuda.CUDAGraph()
59
+ comm = NCCLCommunicator()
60
+ # run something in the default stream to initialize torch engine
61
+ a = torch.ones((4, 4), device=f'cuda:{comm.rank}')
62
+ torch.cuda.synchronize()
63
+ with torch.cuda.graph(graph, stream=comm.stream):
64
+ # operation during the graph capture is recorded but not executed
65
+ # see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#creating-a-graph-using-stream-capture # noqa
66
+ comm.all_reduce(a)
67
+ comm.stream.synchronize()
68
+ assert a.mean().cpu().item() == comm.world_size**0
69
+ graph.replay()
70
+ comm.stream.synchronize()
71
+ assert a.mean().cpu().item() == comm.world_size**1
72
+
73
+
74
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
75
+ reason="Need at least 2 GPUs to run the test.")
76
+def test_pynccl_with_cudagraph():
77
+ distributed_run(worker_fn_with_cudagraph, 2)
78
+
79
+
80
+def test_ncclGetUniqueId():
81
+ unique_id = ncclGetUniqueId()
82
+ # `list(unique_id.internal)` is something like this:
83
+ # [34, -16, 23, 83, 109, -19, 59, 95, 2, 0, -86, 55, 10, -128, 0, 29, 0,
84
+ # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
85
+ # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
86
+ # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
87
+ # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
88
+ # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
89
+ # as long as the function doesn't raise an exception, we're good
90
+ assert unique_id is not None
vllm/executor/ray_gpu_executor.py
+2
-4
@@ -188,11 +188,9 @@ class RayGPUExecutor(ExecutorBase):
188
is_driver_worker=True,
189
)
190
191
- # FIXME(woosuk): We are not properly initializing cupy NCCL when
191
+ # FIXME(woosuk): We are not properly initializing pynccl when
192
# we have multiple nodes.
193
- self._run_workers("init_device",
194
- cupy_port=get_open_port()
195
- if not model_config.enforce_eager else None)
193
+ self._run_workers("init_device")
194
self._run_workers(
195
"load_model",
196
max_concurrent_workers=self.parallel_config.
vllm/model_executor/parallel_utils/communication_op.py
+4
-4
@@ -4,12 +4,12 @@ from typing import Any, Dict, List, Optional, Union
4
import torch
5
from torch.distributed import ProcessGroup
6
7
-from vllm.model_executor.parallel_utils import cupy_utils
7
+from vllm.model_executor.parallel_utils import pynccl_utils
8
from vllm.model_executor.parallel_utils.custom_all_reduce import (
9
custom_all_reduce)
10
from vllm.model_executor.parallel_utils.parallel_state import (
11
get_tensor_model_parallel_group, get_tensor_model_parallel_rank,
12
- get_tensor_model_parallel_world_size, is_cupy_nccl_enabled_for_all_reduce)
12
+ get_tensor_model_parallel_world_size, is_pynccl_enabled_for_all_reduce)
13
14
15
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -30,9 +30,9 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
30
out = custom_all_reduce(input_)
31
if out is not None:
32
return out
33
- if is_cupy_nccl_enabled_for_all_reduce():
33
+ if is_pynccl_enabled_for_all_reduce():
34
# TODO: support multiple parallel groups.
35
- cupy_utils.all_reduce(input_)
35
+ pynccl_utils.all_reduce(input_)
36
else:
37
torch.distributed.all_reduce(input_,
38
group=get_tensor_model_parallel_group())
vllm/model_executor/parallel_utils/cupy_utils.py
deleted
-130
@@ -1,130 +0,0 @@
1
-"""CuPy utilities for all-reduce.
2
-
3
-We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4
-CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5
-CUDA graphs.
6
-
7
-NOTE: We use CuPy 12.3 since CuPy 13.0 does not support Python 3.8.
8
-TODO: Remove this file when torch.distributed.all_reduce is fixed.
9
-"""
10
-import contextlib
11
-
12
-import torch
13
-from torch.distributed import ReduceOp
14
-
15
-try:
16
- import cupy
17
- from cupy.cuda import nccl
18
- from cupyx.distributed import NCCLBackend
19
-except ImportError as e:
20
- cupy = e
21
- nccl = None
22
-
23
- class NCCLBackend:
24
- ...
25
-
26
-
27
-_OP_MAPPING = {
28
- ReduceOp.SUM: "sum",
29
- ReduceOp.PRODUCT: "prod",
30
- ReduceOp.MIN: "min",
31
- ReduceOp.MAX: "max",
32
-}
33
-
34
-
35
-class NCCLBackendWithBFloat16(NCCLBackend):
36
- # This is enough to add bfloat16 support for most operations,
37
- # but broadcast will fail (will require changes in compiled
38
- # cupy code).
39
- def _get_nccl_dtype_and_count(self, array, count=None):
40
- nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
41
- torch_dtype = getattr(array, "_torch_dtype", None)
42
- if torch_dtype is torch.bfloat16:
43
- nccl_dtype = nccl.NCCL_BFLOAT16
44
- return nccl_dtype, count
45
-
46
- def barrier(self) -> None:
47
- raise RuntimeError(
48
- "Currently, CuPy NCCL barrier is not supported since the TCP "
49
- "store is immediately stopped after the initialization.")
50
-
51
-
52
-_NCCL_BACKEND = None
53
-_WORLD_SIZE = 0
54
-
55
-
56
-def is_initialized() -> bool:
57
- """Returns whether the NCCL backend is initialized."""
58
- return _NCCL_BACKEND is not None
59
-
60
-
61
-@contextlib.contextmanager
62
-def set_cupy_stream(stream: torch.cuda.Stream):
63
- """Set the cuda stream for communication"""
64
- cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
65
- stream.device_index)
66
- with cupy_stream:
67
- yield
68
-
69
-
70
-def init_process_group(world_size: int, rank: int, host: str,
71
- port: int) -> None:
72
- """Initializes the CuPy NCCL backend.
73
-
74
- # TODO: handle NCCL timeouts.
75
- """
76
- assert not is_initialized()
77
-
78
- if isinstance(cupy, Exception):
79
- raise ImportError(
80
- "NCCLBackend is not available. Please install cupy.") from cupy
81
-
82
- # TODO(woosuk): Create TP and PP process groups for CuPy.
83
- global _NCCL_BACKEND
84
- global _WORLD_SIZE
85
- assert world_size > 0, f"{world_size=} should be a positive integer"
86
- assert 0 <= rank < world_size, (
87
- f"{rank=} should be a integer between [0, {world_size})")
88
-
89
- cupy.cuda.runtime.setDevice(torch.cuda.current_device())
90
- _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
91
- _WORLD_SIZE = world_size
92
-
93
- # Stop the TCP store to prevent the deadlock issues at termination time.
94
- # FIXME(woosuk): This is hacky. Find a more robust solution.
95
- if rank == 0 and hasattr(_NCCL_BACKEND, "_store"):
96
- _NCCL_BACKEND._store.stop()
97
-
98
-
99
-def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
100
- """All-reduces the input tensor across the process group."""
101
- assert input_.is_cuda, f"{input_} should be a cuda tensor"
102
- # Hack to support bfloat16
103
- torch_dtype = input_.dtype
104
- if torch_dtype is torch.bfloat16:
105
- # We need to view as float16, otherwise
106
- # cupy will fail. This will not change
107
- # the underlying data.
108
- input_ = input_.view(torch.float16)
109
- cupy_input = cupy.asarray(input_)
110
- cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
111
- _NCCL_BACKEND.all_reduce(in_array=cupy_input,
112
- out_array=cupy_input,
113
- op=_OP_MAPPING[op])
114
-
115
-
116
-def destroy_process_group() -> None:
117
- """Destroys the NCCL backend."""
118
- global _NCCL_BACKEND
119
- global _WORLD_SIZE
120
- _NCCL_BACKEND = None
121
- _WORLD_SIZE = 0
122
-
123
-
124
-def get_world_size() -> int:
125
- """Returns the world size."""
126
- return _WORLD_SIZE
127
-
128
-
129
-def get_nccl_backend():
130
- return _NCCL_BACKEND
vllm/model_executor/parallel_utils/parallel_state.py
+18
-18
@@ -7,7 +7,7 @@ import contextlib
7
8
import torch
9
10
-from vllm.model_executor.parallel_utils import cupy_utils
10
+from vllm.model_executor.parallel_utils import pynccl_utils
11
12
# Tensor model parallel group that the current rank belongs to.
13
_TENSOR_MODEL_PARALLEL_GROUP = None
@@ -210,36 +210,36 @@ def destroy_model_parallel():
210
global _PIPELINE_GLOBAL_RANKS
211
_PIPELINE_GLOBAL_RANKS = None
212
213
- # Destroy the cupy states if any.
214
- cupy_utils.destroy_process_group()
213
+ # Destroy the pynccl states if any.
214
+ pynccl_utils.destroy_process_group()
215
216
217
-# Whether to use cupy for nccl all reduce.
218
-# We use cupy for all reduce when using CUDA graph, because torch.distributed
217
+# Whether to use pynccl for nccl all reduce.
218
+# We use pynccl for all reduce when using CUDA graph, because torch.distributed
219
# is not well supported by CUDA graph.
220
-_ENABLE_CUPY_FOR_ALL_REDUCE = False
220
+_ENABLE_PYNCCL_FOR_ALL_REDUCE = False
221
222
223
@contextlib.contextmanager
224
-def with_cupy_nccl_for_all_reduce():
225
- """use CuPy nccl instead of torch.distributed for all reduce"""
224
+def with_pynccl_for_all_reduce():
225
+ """use pynccl instead of torch.distributed for all reduce"""
226
tp_size = get_tensor_model_parallel_world_size()
227
if tp_size == 1:
228
# No-op.
229
- # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
229
+ # NOTE(woosuk): We don't initialize pynccl when tp_size is 1.
230
yield
231
else:
232
- global _ENABLE_CUPY_FOR_ALL_REDUCE
233
- old = _ENABLE_CUPY_FOR_ALL_REDUCE
234
- _ENABLE_CUPY_FOR_ALL_REDUCE = True
232
+ global _ENABLE_PYNCCL_FOR_ALL_REDUCE
233
+ old = _ENABLE_PYNCCL_FOR_ALL_REDUCE
234
+ _ENABLE_PYNCCL_FOR_ALL_REDUCE = True
235
236
stream = torch.cuda.current_stream()
237
- with cupy_utils.set_cupy_stream(stream):
237
+ with pynccl_utils.set_pynccl_stream(stream):
238
yield
239
- _ENABLE_CUPY_FOR_ALL_REDUCE = old
239
+ _ENABLE_PYNCCL_FOR_ALL_REDUCE = old
240
241
242
-def is_cupy_nccl_enabled_for_all_reduce():
243
- """check if CuPy nccl is enabled for all reduce"""
244
- global _ENABLE_CUPY_FOR_ALL_REDUCE
245
- return _ENABLE_CUPY_FOR_ALL_REDUCE
242
+def is_pynccl_enabled_for_all_reduce():
243
+ """check if pynccl is enabled for all reduce"""
244
+ global _ENABLE_PYNCCL_FOR_ALL_REDUCE
245
+ return _ENABLE_PYNCCL_FOR_ALL_REDUCE
vllm/model_executor/parallel_utils/pynccl.py
new
+258
@@ -0,0 +1,258 @@
1
+# This file is a pure Python wrapper for the NCCL library.
2
+# The main purpose is to use NCCL combined with CUDA graph.
3
+# Before writing this script, we tried the following approach:
4
+# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself
5
+# often gets stuck when initializing the NCCL communicator.
6
+# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce`
7
+# contains many other potential cuda APIs, that are not allowed during
8
+# capturing the CUDA graph. For further details, please check
9
+# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ .
10
+#
11
+# Another rejected idea is to write a C/C++ binding for NCCL. It is usually
12
+# doable, but we often encounter issues related with nccl versions, and need
13
+# to switch between different versions of NCCL. See
14
+# https://github.com/NVIDIA/nccl/issues/1234 for more details.
15
+# A C/C++ binding is not flexible enough to handle this. It requires
16
+# recompilation of the code every time we want to switch between different
17
+# versions. This current implementation, with a **pure** Python wrapper, is
18
+# more flexible. We can easily switch between different versions of NCCL by
19
+# changing the environment variable `VLLM_NCCL_SO_PATH`, or the `so_file`
20
+# variable in the code.
21
+
22
+import ctypes
23
+import datetime
24
+import logging
25
+import os
26
+
27
+# ===================== import region =====================
28
+import torch
29
+import torch.distributed as dist
30
+from torch.distributed import ReduceOp
31
+
32
+logger = logging.getLogger(__name__)
33
+
34
+so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
35
+
36
+# manually load the nccl library
37
+if so_file:
38
+ logger.info(
39
+ f"Loading nccl from environment variable VLLM_NCCL_SO_PATH={so_file}")
40
+else:
41
+ if torch.version.cuda is not None:
42
+ so_file = "libnccl.so"
43
+ elif torch.version.hip is not None:
44
+ so_file = "librccl.so"
45
+ else:
46
+ raise ValueError("NCCL only supports CUDA and ROCm backends.")
47
+ logger.debug(f"Loading nccl from library {so_file}")
48
+
49
+try:
50
+ nccl = ctypes.CDLL(so_file)
51
+except Exception as e:
52
+ logger.error(
53
+ f"Failed to load NCCL library from {so_file} ."
54
+ "It is expected if you are not running on NVIDIA/AMD GPUs."
55
+ "Otherwise please set the environment variable VLLM_NCCL_SO_PATH"
56
+ " to point to the correct nccl library path.")
57
+ raise e
58
+
59
+# === export types and functions from nccl to Python ===
60
+# for the original nccl definition, please check
61
+# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in
62
+
63
+ncclResult_t = ctypes.c_int
64
+
65
+# equivalent to c declaration:
66
+# ncclResult_t ncclGetVersion(int *version);
67
+_c_ncclGetVersion = nccl.ncclGetVersion
68
+_c_ncclGetVersion.restype = ctypes.c_int
69
+_c_ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
70
+
71
+
72
+def ncclGetVersion() -> str:
73
+ version = ctypes.c_int()
74
+ result = _c_ncclGetVersion(ctypes.byref(version))
75
+ assert result == 0
76
+ # something like 21903 --> "2.19.3"
77
+ version_str = str(version.value)
78
+ major = version_str[0].lstrip("0")
79
+ minor = version_str[1:3].lstrip("0")
80
+ patch = version_str[3:].lstrip("0")
81
+ return f"{major}.{minor}.{patch}"
82
+
83
+
84
+class NcclUniqueId(ctypes.Structure):
85
+ _fields_ = [("internal", ctypes.c_byte * 128)]
86
+
87
+
88
+# equivalent to c declaration:
89
+# ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId);
90
+_c_ncclGetUniqueId = nccl.ncclGetUniqueId
91
+_c_ncclGetUniqueId.restype = ctypes.c_int
92
+_c_ncclGetUniqueId.argtypes = [ctypes.POINTER(NcclUniqueId)]
93
+
94
+
95
+def ncclGetUniqueId() -> NcclUniqueId:
96
+ unique_id = NcclUniqueId()
97
+ result = _c_ncclGetUniqueId(ctypes.byref(unique_id))
98
+ assert result == 0
99
+ return unique_id
100
+
101
+
102
+# equivalent to c declaration:
103
+# ncclResult_t ncclCommInitRank(
104
+# ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank);
105
+# note that ncclComm_t is a pointer type, so the first argument
106
+# is a pointer to a pointer
107
+_c_ncclCommInitRank = nccl.ncclCommInitRank
108
+_c_ncclCommInitRank.restype = ctypes.c_int
109
+_c_ncclCommInitRank.argtypes = [
110
+ ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, NcclUniqueId, ctypes.c_int
111
+]
112
+
113
+
114
+# enums
115
+class ncclDataType_t(ctypes.c_int):
116
+ ncclInt8 = 0
117
+ ncclChar = 0
118
+ ncclUint8 = 1
119
+ ncclInt32 = 2
120
+ ncclInt = 2
121
+ ncclUint32 = 3
122
+ ncclInt64 = 4
123
+ ncclUint64 = 5
124
+ ncclFloat16 = 6
125
+ ncclHalf = 6
126
+ ncclFloat32 = 7
127
+ ncclFloat = 7
128
+ ncclFloat64 = 8
129
+ ncclDouble = 8
130
+ ncclBfloat16 = 9
131
+ ncclNumTypes = 10
132
+
133
+ @classmethod
134
+ def from_torch(cls, dtype: torch.dtype) -> 'ncclDataType_t':
135
+ if dtype == torch.int8:
136
+ return cls.ncclInt8
137
+ if dtype == torch.uint8:
138
+ return cls.ncclUint8
139
+ if dtype == torch.int32:
140
+ return cls.ncclInt32
141
+ if dtype == torch.int64:
142
+ return cls.ncclInt64
143
+ if dtype == torch.float16:
144
+ return cls.ncclFloat16
145
+ if dtype == torch.float32:
146
+ return cls.ncclFloat32
147
+ if dtype == torch.float64:
148
+ return cls.ncclFloat64
149
+ if dtype == torch.bfloat16:
150
+ return cls.ncclBfloat16
151
+ raise ValueError(f"Unsupported dtype: {dtype}")
152
+
153
+
154
+class ncclRedOp_t(ctypes.c_int):
155
+ ncclSum = 0
156
+ ncclProd = 1
157
+ ncclMax = 2
158
+ ncclMin = 3
159
+ ncclAvg = 4
160
+ ncclNumOps = 5
161
+
162
+ @classmethod
163
+ def from_torch(cls, op: ReduceOp) -> 'ncclRedOp_t':
164
+ if op == ReduceOp.SUM:
165
+ return cls.ncclSum
166
+ if op == ReduceOp.PRODUCT:
167
+ return cls.ncclProd
168
+ if op == ReduceOp.MAX:
169
+ return cls.ncclMax
170
+ if op == ReduceOp.MIN:
171
+ return cls.ncclMin
172
+ if op == ReduceOp.AVG:
173
+ return cls.ncclAvg
174
+ raise ValueError(f"Unsupported op: {op}")
175
+
176
+
177
+# equivalent to c declaration:
178
+# ncclResult_t ncclAllReduce(
179
+# const void* sendbuff, void* recvbuff, size_t count,
180
+# ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
181
+# udaStream_t stream);
182
+# note that cudaStream_t is a pointer type, so the last argument is a pointer
183
+_c_ncclAllReduce = nccl.ncclAllReduce
184
+_c_ncclAllReduce.restype = ctypes.c_int
185
+_c_ncclAllReduce.argtypes = [
186
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclDataType_t,
187
+ ncclRedOp_t, ctypes.c_void_p, ctypes.c_void_p
188
+]
189
+
190
+# equivalent to c declaration:
191
+# ncclResult_t ncclCommDestroy(ncclComm_t comm);
192
+_c_ncclCommDestroy = nccl.ncclCommDestroy
193
+_c_ncclCommDestroy.restype = ctypes.c_int
194
+_c_ncclCommDestroy.argtypes = [ctypes.c_void_p]
195
+
196
+
197
+class NCCLCommunicator:
198
+
199
+ def __init__(
200
+ self,
201
+ backend=None,
202
+ init_method=None,
203
+ timeout=datetime.timedelta(seconds=10),
204
+ world_size: int = -1,
205
+ rank: int = -1,
206
+ store=None,
207
+ group_name: str = "",
208
+ pg_options=None,
209
+ ):
210
+ if not dist.is_initialized():
211
+ backend = backend or "nccl"
212
+ assert backend == 'nccl', (
213
+ "only use nccl backend for starting the NCCL communicator")
214
+ dist.init_process_group(backend=backend,
215
+ init_method=init_method,
216
+ timeout=timeout,
217
+ world_size=world_size,
218
+ rank=rank,
219
+ store=store,
220
+ group_name=group_name,
221
+ pg_options=pg_options)
222
+ self.world_size = dist.get_world_size()
223
+ self.rank = dist.get_rank()
224
+ torch.cuda.set_device(self.rank)
225
+ if self.rank == 0:
226
+ self.unique_id = ncclGetUniqueId()
227
+ else:
228
+ self.unique_id = NcclUniqueId()
229
+ tensor = torch.ByteTensor(list(self.unique_id.internal)).cuda(
230
+ self.rank)
231
+ dist.broadcast(tensor, src=0)
232
+ byte_list = tensor.cpu().tolist()
233
+ self.unique_id = NcclUniqueId()
234
+ for i, byte in enumerate(byte_list):
235
+ self.unique_id.internal[i] = byte
236
+ self.comm = ctypes.c_void_p()
237
+ result = _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
238
+ self.unique_id, self.rank)
239
+ assert result == 0
240
+ self.stream = torch.cuda.Stream(device=f"cuda:{self.rank}")
241
+
242
+ def all_reduce(self,
243
+ tensor: torch.Tensor,
244
+ op: ReduceOp = ReduceOp.SUM,
245
+ stream=None):
246
+ if stream is None:
247
+ stream = self.stream
248
+ result = _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
249
+ ctypes.c_void_p(tensor.data_ptr()),
250
+ tensor.numel(),
251
+ ncclDataType_t.from_torch(tensor.dtype),
252
+ ncclRedOp_t.from_torch(op), self.comm,
253
+ ctypes.c_void_p(stream.cuda_stream))
254
+ assert result == 0
255
+
256
+ def __del__(self):
257
+ dist.destroy_process_group()
258
+ _c_ncclCommDestroy(self.comm)
vllm/model_executor/parallel_utils/pynccl_utils.py
new
+64
@@ -0,0 +1,64 @@
1
+import contextlib
2
+import logging
3
+from typing import Optional
4
+
5
+import torch
6
+from torch.distributed import ReduceOp
7
+
8
+logger = logging.getLogger(__name__)
9
+
10
+try:
11
+ from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
12
+ ncclGetVersion)
13
+ logger.info(f"vLLM is using nccl=={ncclGetVersion()}")
14
+except Exception as e:
15
+ # in non-NVIDIA environments, we can't import the nccl module
16
+ # e.g. when running on machines with AMD GPUs
17
+ logger.info(f"Failed to import NCCL library: {e}")
18
+ logger.info("It is expected if you are not running on NVIDIA GPUs.")
19
+ pass
20
+
21
+comm: Optional["NCCLCommunicator"] = None
22
+
23
+
24
+def is_initialized() -> bool:
25
+ """Returns whether the NCCL backend is initialized."""
26
+ return comm is not None
27
+
28
+
29
+@contextlib.contextmanager
30
+def set_pynccl_stream(stream: torch.cuda.Stream):
31
+ """Set the cuda stream for communication"""
32
+ try:
33
+ comm.stream = stream
34
+ yield
35
+ finally:
36
+ pass
37
+
38
+
39
+def init_process_group(world_size: int, rank: int, init_method: str) -> None:
40
+ assert not is_initialized()
41
+ global comm
42
+ comm = NCCLCommunicator(init_method=init_method,
43
+ world_size=world_size,
44
+ rank=rank)
45
+
46
+
47
+def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
48
+ """All-reduces the input tensor across the process group."""
49
+ assert input_.is_cuda, f"{input_} should be a cuda tensor"
50
+ comm.all_reduce(input_, op)
51
+
52
+
53
+def destroy_process_group() -> None:
54
+ global comm
55
+ comm = None
56
+
57
+
58
+def get_world_size() -> int:
59
+ """Returns the world size."""
60
+ return comm.world_size
61
+
62
+
63
+def get_nccl_backend():
64
+ return comm
vllm/test_utils.py
+1
-4
@@ -16,10 +16,7 @@ def init_test_distributed_environment(
16
worker_use_ray=True)
17
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
18
init_distributed_environment(
19
- parallel_config,
20
- rank,
21
- cupy_port=None,
22
- distributed_init_method=distributed_init_method)
19
+ parallel_config, rank, distributed_init_method=distributed_init_method)
20
21
22
def multi_process_tensor_parallel(
vllm/worker/model_runner.py
+16
-13
@@ -15,11 +15,11 @@ from vllm.lora.request import LoRARequest
15
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
16
from vllm.model_executor import SamplingMetadata
17
from vllm.model_executor.model_loader import get_model
18
-from vllm.model_executor.parallel_utils import cupy_utils, custom_all_reduce
18
+from vllm.model_executor.parallel_utils import custom_all_reduce, pynccl_utils
19
from vllm.model_executor.parallel_utils.communication_op import (
20
broadcast_tensor_dict)
21
from vllm.model_executor.parallel_utils.parallel_state import (
22
- with_cupy_nccl_for_all_reduce)
22
+ with_pynccl_for_all_reduce)
23
from vllm.sampling_params import SamplingParams, SamplingType
24
from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
25
SequenceGroupMetadata)
@@ -764,7 +764,7 @@ class ModelRunner:
764
"""
765
# NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
766
# deleted before the CUDA graphs.
767
- self.cupy_nccl_backend = cupy_utils.get_nccl_backend()
767
+ self.pynccl_backend = pynccl_utils.get_nccl_backend()
768
769
assert not self.model_config.enforce_eager
770
logger.info("Capturing the model for CUDA graphs. This may lead to "
@@ -794,11 +794,11 @@ class ModelRunner:
794
]
795
796
# NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
797
- # kernel, CuPy NCCL, and PyTorch NCCL. When using CUDA graph, we use
798
- # either custom all-reduce kernel or CuPy NCCL. When not using CUDA
797
+ # kernel, pynccl, and PyTorch NCCL. When using CUDA graph, we use
798
+ # either custom all-reduce kernel or pynccl. When not using CUDA
799
# graph, we use either custom all-reduce kernel or PyTorch NCCL.
800
# We always prioritize using custom all-reduce kernel but fall back
801
- # to PyTorch or CuPy NCCL if it is disabled or not supported.
801
+ # to PyTorch or pynccl if it is disabled or not supported.
802
with custom_all_reduce.capture():
803
# NOTE: Capturing the largest batch size first may help reduce the
804
# memory usage of CUDA graph.
@@ -846,12 +846,14 @@ class ModelRunner:
846
logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
847
848
def __del__(self) -> None:
849
- # Delete the CUDA graphs before deleting the CuPy NCCL communicator.
849
+ # Delete the CUDA graphs before deleting the pynccl communicator.
850
# NOTE(woosuk): This is necessary because otherwise deadlocks can
851
# happen.
852
# FIXME(woosuk): This is a bit hacky. Find a more robust solution.
853
+ # TODO(youkaichao): when we get enough user feedback that pynccl is
854
+ # more stable than cupy, we can remove this, e.g. in v0.4.1.
855
self.graph_runners.clear()
854
- self.cupy_nccl_backend = None
856
+ self.pynccl_backend = None
857
858
@property
859
def vocab_size(self) -> int:
@@ -879,7 +881,7 @@ class CUDAGraphRunner:
881
# Run the model once without capturing the graph.
882
# This is to make sure that the captured graph does not include the
883
# kernel launches for initial benchmarking (e.g., Triton autotune).
882
- with _maybe_cupy_nccl():
884
+ with _maybe_pynccl():
885
self.model(
886
input_ids,
887
positions,
@@ -894,7 +896,7 @@ class CUDAGraphRunner:
896
# https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
897
self.graph = torch.cuda.CUDAGraph()
898
with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
897
- with _maybe_cupy_nccl():
899
+ with _maybe_pynccl():
900
hidden_states = self.model(
901
input_ids,
902
positions,
@@ -947,9 +949,10 @@ class CUDAGraphRunner:
949
950
951
@contextlib.contextmanager
950
-def _maybe_cupy_nccl():
951
- if cupy_utils.is_initialized() and not custom_all_reduce.is_initialized():
952
- with with_cupy_nccl_for_all_reduce():
952
+def _maybe_pynccl():
953
+ if pynccl_utils.is_initialized(
954
+ ) and not custom_all_reduce.is_initialized():
955
+ with with_pynccl_for_all_reduce():
956
yield
957
else:
958
yield
vllm/worker/worker.py
+14
-16
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11
from vllm.lora.request import LoRARequest
12
from vllm.model_executor import set_random_seed
13
-from vllm.model_executor.parallel_utils import cupy_utils
13
+from vllm.model_executor.parallel_utils import pynccl_utils
14
from vllm.model_executor.parallel_utils.communication_op import (
15
broadcast_tensor_dict)
16
from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
@@ -75,7 +75,7 @@ class Worker:
75
self.cache_engine = None
76
self.gpu_cache = None
77
78
- def init_device(self, cupy_port: Optional[int] = None) -> None:
78
+ def init_device(self) -> None:
79
if self.device_config.device.type == "cuda":
80
# torch.distributed.all_reduce does not free the input tensor until
81
# the synchronization point. This causes the memory usage to grow
@@ -98,7 +98,7 @@ class Worker:
98
f"Not support device type: {self.device_config.device}")
99
# Initialize the distributed environment.
100
init_distributed_environment(self.parallel_config, self.rank,
101
- cupy_port, self.distributed_init_method)
101
+ self.distributed_init_method)
102
# Set random seed.
103
set_random_seed(self.model_config.seed)
104
@@ -250,7 +250,6 @@ class Worker:
250
def init_distributed_environment(
251
parallel_config: ParallelConfig,
252
rank: int,
253
- cupy_port: Optional[int],
253
distributed_init_method: Optional[str] = None,
254
) -> None:
255
"""Initialize the distributed environment."""
@@ -273,28 +272,27 @@ def init_distributed_environment(
272
init_method=distributed_init_method,
273
)
274
276
- if cupy_utils.is_initialized():
277
- cupy_world_size = cupy_utils.get_world_size()
278
- if cupy_world_size != parallel_config.world_size:
275
+ if pynccl_utils.is_initialized():
276
+ pynccl_world_size = pynccl_utils.get_world_size()
277
+ if pynccl_world_size != parallel_config.world_size:
278
raise RuntimeError(
280
- "cupy.distributed is already initialized but the cupy world "
279
+ "pynccl is already initialized but the pynccl world "
280
"size does not match parallel_config.world_size "
282
- f"({cupy_world_size} vs. {parallel_config.world_size}).")
283
- elif (parallel_config.world_size > 1 and cupy_port is not None):
284
- # NOTE(woosuk): We don't initialize CuPy process group when world size
281
+ f"({pynccl_world_size} vs. {parallel_config.world_size}).")
282
+ elif parallel_config.world_size > 1:
283
+ # NOTE(woosuk): We don't initialize pynccl process group when world size
284
# is 1.
285
# TODO(woosuk): Support multi-node connection.
287
- cupy_utils.init_process_group(
286
+ pynccl_utils.init_process_group(
287
world_size=parallel_config.world_size,
288
rank=rank,
290
- host="localhost",
291
- port=cupy_port,
289
+ init_method=distributed_init_method,
290
)
291
292
# A small all_reduce for warmup.
293
torch.distributed.all_reduce(torch.zeros(1).cuda())
296
- if cupy_utils.is_initialized():
297
- cupy_utils.all_reduce(torch.zeros(1).cuda())
294
+ if pynccl_utils.is_initialized():
295
+ pynccl_utils.all_reduce(torch.zeros(1).cuda())
296
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
297
parallel_config.pipeline_parallel_size)
298