@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 208b71bcc1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] refactor pynccl (#4591)

[Core][Distributed] refactor pynccl to hold multiple communicators (#4591)

youkaichao committed May 9, 2024 at 19:48 UTC 208b71bcc1b94df1fdd2fc10da3e04c706340188
8 files changed +467 -434
tests/distributed/test_pynccl.py
+42 -36
@@ -1,15 +1,15 @@
1 import multiprocessing
2 +import os
3
4 import pytest
5 import torch
6
6 -import vllm.distributed.device_communicators.pynccl_utils as pynccl_utils
7 -from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
8 -from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
9 - ncclGetUniqueId)
10 -from vllm.distributed.parallel_state import (
11 - ensure_model_parallel_initialized, get_tensor_model_parallel_cpu_group,
12 - init_distributed_environment, with_pynccl_for_all_reduce)
7 +from vllm.distributed.communication_op import ( # noqa
8 + graph_capture_mode, tensor_model_parallel_all_reduce)
9 +from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
10 +from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
11 +from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
12 + init_distributed_environment)
13 from vllm.utils import update_environment_variables
14
15
@@ -41,6 +41,9 @@ def worker_fn_wrapper(fn):
41 # and update the environment variables in the function
42 def wrapped_fn(env):
43 update_environment_variables(env)
44 + local_rank = os.environ['LOCAL_RANK']
45 + device = torch.device(f"cuda:{local_rank}")
46 + torch.cuda.set_device(device)
47 init_distributed_environment()
48 fn()
49
@@ -49,11 +52,13 @@ def worker_fn_wrapper(fn):
52
53 @worker_fn_wrapper
54 def worker_fn():
52 - comm = NCCLCommunicator()
53 - tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
54 - comm.all_reduce(tensor)
55 + pynccl_comm = PyNcclCommunicator()
56 + tensor = torch.ones(16, 1024, 1024,
57 + dtype=torch.float32).cuda(pynccl_comm.rank)
58 + with pynccl_comm.change_state(enable=True):
59 + pynccl_comm.all_reduce(tensor)
60 result = tensor.mean().cpu().item()
56 - assert result == comm.world_size
61 + assert result == pynccl_comm.world_size
62
63
64 @pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -70,37 +75,35 @@ def multiple_tp_worker_fn():
75 torch.distributed.new_group(ranks=[2, 3], backend="gloo")
76 ]
77 group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
73 - comm = NCCLCommunicator(group=group, device=device)
78 + pynccl_comm = PyNcclCommunicator(group=group, device=device)
79 tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
75 - # two groups can communicate independently
76 - if torch.distributed.get_rank() in [0, 1]:
77 - comm.all_reduce(tensor)
78 - comm.all_reduce(tensor)
79 - result = tensor.mean().cpu().item()
80 - assert result == 4
81 - else:
82 - comm.all_reduce(tensor)
83 - result = tensor.mean().cpu().item()
84 - assert result == 2
80 + with pynccl_comm.change_state(enable=True):
81 + # two groups can communicate independently
82 + if torch.distributed.get_rank() in [0, 1]:
83 + pynccl_comm.all_reduce(tensor)
84 + pynccl_comm.all_reduce(tensor)
85 + result = tensor.mean().cpu().item()
86 + assert result == 4
87 + else:
88 + pynccl_comm.all_reduce(tensor)
89 + result = tensor.mean().cpu().item()
90 + assert result == 2
91
92
93 @pytest.mark.skipif(torch.cuda.device_count() < 4,
94 reason="Need at least 4 GPUs to run the test.")
95 def test_pynccl_multiple_tp():
96 # this tests pynccl for multiple tp groups, in a standalone way
91 - # i.e. call `comm.all_reduce` directly
97 + # i.e. call `pynccl_comm.all_reduce` directly
98 distributed_run(multiple_tp_worker_fn, 4)
99
100
101 @worker_fn_wrapper
102 def multiple_tp_with_vllm_worker_fn():
103 device = torch.device(f"cuda:{torch.distributed.get_rank()}")
98 - torch.cuda.set_device(torch.distributed.get_rank())
104 ensure_model_parallel_initialized(2, 2)
100 - pynccl_utils.init_process_group(
101 - group=get_tensor_model_parallel_cpu_group())
105 tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
103 - with with_pynccl_for_all_reduce():
106 + with graph_capture_mode():
107 # two tp groups can communicate independently
108 if torch.distributed.get_rank() in [0, 1]:
109 tensor = tensor_model_parallel_all_reduce(tensor)
@@ -125,19 +128,21 @@ def test_pynccl_multiple_tp_with_vllm():
128 def worker_fn_with_cudagraph():
129 with torch.no_grad():
130 graph = torch.cuda.CUDAGraph()
128 - comm = NCCLCommunicator()
131 + pynccl_comm = PyNcclCommunicator()
132 # run something in the default stream to initialize torch engine
130 - a = torch.ones((4, 4), device=f'cuda:{comm.rank}')
133 + a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
134 torch.cuda.synchronize()
132 - with torch.cuda.graph(graph, stream=comm.stream):
135 + with torch.cuda.graph(
136 + graph, stream=pynccl_comm.stream), pynccl_comm.change_state(
137 + enable=True):
138 # operation during the graph capture is recorded but not executed
139 # see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#creating-a-graph-using-stream-capture # noqa
135 - comm.all_reduce(a)
136 - comm.stream.synchronize()
137 - assert a.mean().cpu().item() == comm.world_size**0
140 + pynccl_comm.all_reduce(a)
141 + pynccl_comm.stream.synchronize()
142 + assert a.mean().cpu().item() == pynccl_comm.world_size**0
143 graph.replay()
139 - comm.stream.synchronize()
140 - assert a.mean().cpu().item() == comm.world_size**1
144 + pynccl_comm.stream.synchronize()
145 + assert a.mean().cpu().item() == pynccl_comm.world_size**1
146
147
148 @pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -147,7 +152,8 @@ def test_pynccl_with_cudagraph():
152
153
154 def test_ncclGetUniqueId():
150 - unique_id = ncclGetUniqueId()
155 + lib = NCCLLibrary()
156 + unique_id = lib.ncclGetUniqueId()
157 # `list(unique_id.internal)` is something like this:
158 # [34, -16, 23, 83, 109, -19, 59, 95, 2, 0, -86, 55, 10, -128, 0, 29, 0,
159 # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
vllm/distributed/communication_op.py
+24 -4
@@ -1,4 +1,5 @@
1 from collections import namedtuple
2 +from contextlib import contextmanager
3 from typing import Any, Dict, List, Optional, Tuple, Union
4
5 import torch
@@ -8,7 +9,26 @@ from .parallel_state import (get_cpu_world_group,
9 get_tensor_model_parallel_group,
10 get_tensor_model_parallel_rank,
11 get_tensor_model_parallel_world_size,
11 - is_pynccl_enabled_for_all_reduce)
12 + get_tp_pynccl_communicator)
13 +
14 +
15 +@contextmanager
16 +def graph_capture_mode():
17 + # In graph capture, we have to be very careful about the collective
18 + # operations. The current status is:
19 + # allreduce \ Mode | Eager | Graph |
20 + # --------------------------------------------
21 + # custom allreduce | enabled | enabled |
22 + # PyNccl | disabled| enabled |
23 + # torch.distributed | enabled | disabled|
24 + #
25 + # Note that custom allreduce will have a runtime check, if the tensor size
26 + # is too large, it will fallback to the next available option.
27 + pynccl_comm = get_tp_pynccl_communicator()
28 + assert pynccl_comm is not None
29 + with pynccl_comm.change_state(enable=True,
30 + stream=torch.cuda.current_stream()):
31 + yield
32
33
34 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -23,7 +43,6 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
43 TLDR: always assume this function modifies its input, but use the return
44 value as the output.
45 """
26 - from vllm.distributed.device_communicators import pynccl_utils
46 from vllm.distributed.device_communicators.custom_all_reduce import (
47 custom_all_reduce)
48
@@ -33,8 +52,9 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
52 out = custom_all_reduce(input_)
53 if out is not None:
54 return out
36 - if is_pynccl_enabled_for_all_reduce():
37 - pynccl_utils.all_reduce(input_)
55 + pynccl_comm = get_tp_pynccl_communicator()
56 + if (pynccl_comm is not None and not pynccl_comm.disabled):
57 + pynccl_comm.all_reduce(input_)
58 else:
59 torch.distributed.all_reduce(input_,
60 group=get_tensor_model_parallel_group())
vllm/distributed/device_communicators/pynccl.py
+78 -216
@@ -1,26 +1,4 @@
1 -# This file is a pure Python wrapper for the NCCL library.
2 -# The main purpose is to use NCCL combined with CUDA graph.
3 -# Before writing this script, we tried the following approach:
4 -# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself
5 -# often gets stuck when initializing the NCCL communicator.
6 -# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce`
7 -# contains many other potential cuda APIs, that are not allowed during
8 -# capturing the CUDA graph. For further details, please check
9 -# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ .
10 -#
11 -# Another rejected idea is to write a C/C++ binding for NCCL. It is usually
12 -# doable, but we often encounter issues related with nccl versions, and need
13 -# to switch between different versions of NCCL. See
14 -# https://github.com/NVIDIA/nccl/issues/1234 for more details.
15 -# A C/C++ binding is not flexible enough to handle this. It requires
16 -# recompilation of the code every time we want to switch between different
17 -# versions. This current implementation, with a **pure** Python wrapper, is
18 -# more flexible. We can easily switch between different versions of NCCL by
19 -# changing the environment variable `VLLM_NCCL_SO_PATH`, or the `so_file`
20 -# variable in the code.
21 -
22 -import ctypes
23 -import platform
1 +from contextlib import contextmanager
2 from typing import Optional, Union
3
4 # ===================== import region =====================
@@ -28,217 +6,70 @@ import torch
6 import torch.distributed as dist
7 from torch.distributed import ProcessGroup, ReduceOp
8
9 +from vllm.distributed.device_communicators.pynccl_wrapper import (
10 + NCCLLibrary, buffer_type, cudaStream_t, ncclComm_t, ncclDataTypeEnum,
11 + ncclRedOpTypeEnum, ncclUniqueId)
12 from vllm.distributed.parallel_state import get_cpu_world_group, get_local_rank
13 from vllm.logger import init_logger
33 -from vllm.utils import find_nccl_library, nccl_integrity_check
14
15 logger = init_logger(__name__)
16
37 -so_file = find_nccl_library()
38 -
39 -try:
40 - # load the library in another process.
41 - # if it core dumps, it will not crash the current process
42 - nccl_integrity_check(so_file)
43 - nccl = ctypes.CDLL(so_file)
44 -except Exception as e:
45 - logger.error(
46 - "Failed to load NCCL library from %s ."
47 - "It is expected if you are not running on NVIDIA/AMD GPUs."
48 - "Otherwise, the nccl library might not exist, be corrupted "
49 - "or it does not support the current platform %s."
50 - "One solution is to download libnccl2 version 2.18 from "
51 - "https://developer.download.nvidia.com/compute/cuda/repos/ "
52 - "and extract the libnccl.so.2 file. If you already have the "
53 - "library, please set the environment variable VLLM_NCCL_SO_PATH"
54 - " to point to the correct nccl library path.", so_file,
55 - platform.platform())
56 - raise e
57 -
58 -# === export types and functions from nccl to Python ===
59 -# for the original nccl definition, please check
60 -# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in
61 -
62 -ncclResult_t = ctypes.c_int
63 -
64 -_c_ncclGetErrorString = nccl.ncclGetErrorString
65 -_c_ncclGetErrorString.restype = ctypes.c_char_p
66 -_c_ncclGetErrorString.argtypes = [ncclResult_t]
67 -
68 -
69 -def NCCL_CHECK(result: ncclResult_t) -> None:
70 - if result != 0:
71 - error_str = _c_ncclGetErrorString(result)
72 - error_str = error_str.decode("utf-8")
73 - raise RuntimeError(f"NCCL error: {error_str}")
74 -
75 -
76 -# equivalent to c declaration:
77 -# ncclResult_t ncclGetVersion(int *version);
78 -_c_ncclGetVersion = nccl.ncclGetVersion
79 -_c_ncclGetVersion.restype = ctypes.c_int
80 -_c_ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
81 -
82 -
83 -def ncclGetVersion() -> str:
84 - version = ctypes.c_int()
85 - NCCL_CHECK(_c_ncclGetVersion(ctypes.byref(version)))
86 - # something like 21903 --> "2.19.3"
87 - version_str = str(version.value)
88 - major = version_str[0].lstrip("0")
89 - minor = version_str[1:3].lstrip("0")
90 - patch = version_str[3:].lstrip("0")
91 - return f"{major}.{minor}.{patch}"
92 -
93 -
94 -class NcclUniqueId(ctypes.Structure):
95 - _fields_ = [("internal", ctypes.c_byte * 128)]
96 -
97 -
98 -# equivalent to c declaration:
99 -# ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId);
100 -_c_ncclGetUniqueId = nccl.ncclGetUniqueId
101 -_c_ncclGetUniqueId.restype = ctypes.c_int
102 -_c_ncclGetUniqueId.argtypes = [ctypes.POINTER(NcclUniqueId)]
103 -
104 -
105 -def ncclGetUniqueId() -> NcclUniqueId:
106 - unique_id = NcclUniqueId()
107 - NCCL_CHECK(_c_ncclGetUniqueId(ctypes.byref(unique_id)))
108 - return unique_id
109 -
110 -
111 -# equivalent to c declaration:
112 -# ncclResult_t ncclCommInitRank(
113 -# ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank);
114 -# note that ncclComm_t is a pointer type, so the first argument
115 -# is a pointer to a pointer
116 -_c_ncclCommInitRank = nccl.ncclCommInitRank
117 -_c_ncclCommInitRank.restype = ctypes.c_int
118 -_c_ncclCommInitRank.argtypes = [
119 - ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, NcclUniqueId, ctypes.c_int
120 -]
121 -
122 -ncclDataType_t = ctypes.c_int
123 -
124 -
125 -class ncclDataTypeEnum:
126 - ncclInt8 = 0
127 - ncclChar = 0
128 - ncclUint8 = 1
129 - ncclInt32 = 2
130 - ncclInt = 2
131 - ncclUint32 = 3
132 - ncclInt64 = 4
133 - ncclUint64 = 5
134 - ncclFloat16 = 6
135 - ncclHalf = 6
136 - ncclFloat32 = 7
137 - ncclFloat = 7
138 - ncclFloat64 = 8
139 - ncclDouble = 8
140 - ncclBfloat16 = 9
141 - ncclNumTypes = 10
17
143 - @classmethod
144 - def from_torch(cls, dtype: torch.dtype) -> int:
145 - if dtype == torch.int8:
146 - return cls.ncclInt8
147 - if dtype == torch.uint8:
148 - return cls.ncclUint8
149 - if dtype == torch.int32:
150 - return cls.ncclInt32
151 - if dtype == torch.int64:
152 - return cls.ncclInt64
153 - if dtype == torch.float16:
154 - return cls.ncclFloat16
155 - if dtype == torch.float32:
156 - return cls.ncclFloat32
157 - if dtype == torch.float64:
158 - return cls.ncclFloat64
159 - if dtype == torch.bfloat16:
160 - return cls.ncclBfloat16
161 - raise ValueError(f"Unsupported dtype: {dtype}")
162 -
163 -
164 -ncclRedOp_t = ctypes.c_int
165 -
166 -
167 -class ncclRedOpTypeEnum:
168 - ncclSum = 0
169 - ncclProd = 1
170 - ncclMax = 2
171 - ncclMin = 3
172 - ncclAvg = 4
173 - ncclNumOps = 5
174 -
175 - @classmethod
176 - def from_torch(cls, op: ReduceOp) -> int:
177 - if op == ReduceOp.SUM:
178 - return cls.ncclSum
179 - if op == ReduceOp.PRODUCT:
180 - return cls.ncclProd
181 - if op == ReduceOp.MAX:
182 - return cls.ncclMax
183 - if op == ReduceOp.MIN:
184 - return cls.ncclMin
185 - if op == ReduceOp.AVG:
186 - return cls.ncclAvg
187 - raise ValueError(f"Unsupported op: {op}")
188 -
189 -
190 -# equivalent to c declaration:
191 -# ncclResult_t ncclAllReduce(
192 -# const void* sendbuff, void* recvbuff, size_t count,
193 -# ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
194 -# udaStream_t stream);
195 -# note that cudaStream_t is a pointer type, so the last argument is a pointer
196 -_c_ncclAllReduce = nccl.ncclAllReduce
197 -_c_ncclAllReduce.restype = ctypes.c_int
198 -_c_ncclAllReduce.argtypes = [
199 - ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclRedOp_t,
200 - ncclDataType_t, ctypes.c_void_p, ctypes.c_void_p
201 -]
202 -
203 -# be cautious! this is a collective call, it will block until all
204 -# processes in the communicator have called this function.
205 -# because Python object destruction can happen in random order,
206 -# it is better not to call it at all.
207 -# equivalent to c declaration:
208 -# ncclResult_t ncclCommDestroy(ncclComm_t comm);
209 -_c_ncclCommDestroy = nccl.ncclCommDestroy
210 -_c_ncclCommDestroy.restype = ctypes.c_int
211 -_c_ncclCommDestroy.argtypes = [ctypes.c_void_p]
212 -
213 -
214 -class NCCLCommunicator:
18 +class PyNcclCommunicator:
19
20 def __init__(
21 self,
22 group: Optional[ProcessGroup] = None,
23 device: Optional[Union[int, str, torch.device]] = None,
24 + library_path: Optional[str] = None,
25 ):
26 """
27 Args:
28 group: the process group to work on. If None, it will use the
29 default process group.
225 - device: the device to bind the NCCLCommunicator to. If None,
30 + device: the device to bind the PyNcclCommunicator to. If None,
31 it will be bind to f"cuda:{local_rank}".
32 + library_path: the path to the NCCL library. If None, it will
33 + use the default library path.
34 It is the caller's responsibility to make sure each communicator
35 is bind to a unique device.
36 """
37 assert dist.is_initialized()
38 group = get_cpu_world_group() if group is None else group
39 assert dist.get_backend(group) != dist.Backend.NCCL, (
233 - "NCCLCommunicator should be attached to a non-NCCL group.")
40 + "PyNcclCommunicator should be attached to a non-NCCL group.")
41 self.group = group
42 # note: this rank is the rank in the group
43 self.rank = dist.get_rank(group)
44 self.world_size = dist.get_world_size(group)
45 +
46 + # if world_size == 1, no need to create communicator
47 + if self.world_size == 1:
48 + self.available = False
49 + self.disabled = True
50 + self.stream = None
51 + return
52 + try:
53 + self.nccl = NCCLLibrary(library_path)
54 + except Exception:
55 + # disable because of missing NCCL library
56 + # e.g. in a non-GPU environment
57 + self.available = False
58 + self.disabled = True
59 + self.stream = None
60 + return
61 +
62 + self.available = True
63 + self.disabled = False
64 +
65 + logger.info("vLLM is using nccl==%s", self.nccl.ncclGetVersion())
66 +
67 if self.rank == 0:
239 - self.unique_id = ncclGetUniqueId()
68 + # get the unique id from NCCL
69 + self.unique_id = self.nccl.ncclGetUniqueId()
70 else:
241 - self.unique_id = NcclUniqueId()
71 + # construct an empty unique id
72 + self.unique_id = ncclUniqueId()
73 tensor = torch.ByteTensor(list(self.unique_id.internal))
74 ranks = dist.get_process_group_ranks(group)
75 # arg `src` in `broadcast` is the global rank
@@ -246,7 +77,6 @@ class NCCLCommunicator:
77 byte_list = tensor.tolist()
78 for i, byte in enumerate(byte_list):
79 self.unique_id.internal[i] = byte
249 - self.comm = ctypes.c_void_p()
80 if device is None:
81 local_rank = get_local_rank()
82 device = torch.device(f"cuda:{local_rank}")
@@ -261,15 +91,25 @@ class NCCLCommunicator:
91 # `torch.cuda.device` is a context manager that changes the
92 # current cuda device to the specified one
93 with torch.cuda.device(device):
264 - NCCL_CHECK(
265 - _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
266 - self.unique_id, self.rank))
94 + self.comm: ncclComm_t = self.nccl.ncclCommInitRank(
95 + self.world_size, self.unique_id, self.rank)
96 self.stream = torch.cuda.Stream()
97
98 + # A small all_reduce for warmup.
99 + self.all_reduce(torch.zeros(1, device=device))
100 + self.stream.synchronize()
101 +
102 + # by default it is disabled, e.g. in profiling models and prefill phase.
103 + # to use it, use under `with obj.change_state(enable=True)`, usually
104 + # when we are using CUDA graph.
105 + self.disabled = True
106 +
107 def all_reduce(self,
108 tensor: torch.Tensor,
109 op: ReduceOp = ReduceOp.SUM,
110 stream=None):
111 + if self.disabled:
112 + return
113 # nccl communicator created on a specific device
114 # will only work on tensors on the same device
115 # otherwise it will cause "illegal memory access"
@@ -278,10 +118,32 @@ class NCCLCommunicator:
118 f"but the input tensor is on {tensor.device}")
119 if stream is None:
120 stream = self.stream
281 - NCCL_CHECK(
282 - _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
283 - ctypes.c_void_p(tensor.data_ptr()),
284 - tensor.numel(),
285 - ncclDataTypeEnum.from_torch(tensor.dtype),
286 - ncclRedOpTypeEnum.from_torch(op), self.comm,
287 - ctypes.c_void_p(stream.cuda_stream)))
121 + self.nccl.ncclAllReduce(buffer_type(tensor.data_ptr()),
122 + buffer_type(tensor.data_ptr()), tensor.numel(),
123 + ncclDataTypeEnum.from_torch(tensor.dtype),
124 + ncclRedOpTypeEnum.from_torch(op), self.comm,
125 + cudaStream_t(stream.cuda_stream))
126 +
127 + @contextmanager
128 + def change_state(self,
129 + enable: Optional[bool] = None,
130 + stream: Optional[torch.cuda.Stream] = None):
131 + """
132 + A context manager to change the state of the communicator.
133 + """
134 + if enable is None:
135 + # guess a default value when not specified
136 + enable = self.available
137 +
138 + if stream is None:
139 + stream = self.stream
140 +
141 + old_disable = self.disabled
142 + old_stream = self.stream
143 +
144 + self.stream = stream
145 + self.disabled = not enable
146 + yield
147 +
148 + self.disabled = old_disable
149 + self.stream = old_stream
vllm/distributed/device_communicators/pynccl_utils.py deleted
-66
@@ -1,66 +0,0 @@
1 -import contextlib
2 -from typing import Optional
3 -
4 -import torch
5 -from torch.distributed import ProcessGroup, ReduceOp
6 -
7 -from vllm.logger import init_logger
8 -
9 -logger = init_logger(__name__)
10 -
11 -try:
12 - from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
13 - ncclGetVersion)
14 -except Exception as e:
15 - # in non-NVIDIA environments, we can't import the nccl module
16 - # e.g. when running on machines with AMD GPUs
17 - logger.info("Failed to import NCCL library: %s", e)
18 - logger.info("It is expected if you are not running on NVIDIA GPUs.")
19 - pass
20 -
21 -comm: Optional["NCCLCommunicator"] = None
22 -
23 -
24 -def is_initialized() -> bool:
25 - """Returns whether the NCCL backend is initialized."""
26 - return comm is not None
27 -
28 -
29 -@contextlib.contextmanager
30 -def set_pynccl_stream(stream: torch.cuda.Stream):
31 - """Set the cuda stream for communication"""
32 - try:
33 - assert comm is not None
34 - comm.stream = stream
35 - yield
36 - finally:
37 - pass
38 -
39 -
40 -def init_process_group(group: Optional[ProcessGroup] = None) -> None:
41 - assert not is_initialized()
42 - global comm
43 - logger.info("vLLM is using nccl==%s", ncclGetVersion())
44 - comm = NCCLCommunicator(group=group)
45 -
46 -
47 -def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
48 - """All-reduces the input tensor across the process group."""
49 - assert input_.is_cuda, f"{input_} should be a cuda tensor"
50 - assert comm is not None
51 - comm.all_reduce(input_, op)
52 -
53 -
54 -def destroy_process_group() -> None:
55 - global comm
56 - comm = None
57 -
58 -
59 -def get_world_size() -> int:
60 - """Returns the world size."""
61 - assert comm is not None
62 - return comm.world_size
63 -
64 -
65 -def get_nccl_backend() -> Optional["NCCLCommunicator"]:
66 - return comm
vllm/distributed/device_communicators/pynccl_wrapper.py new
+258
@@ -0,0 +1,258 @@
1 +# This file is a pure Python wrapper for the NCCL library.
2 +# The main purpose is to use NCCL combined with CUDA graph.
3 +# Before writing this script, we tried the following approach:
4 +# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself
5 +# often gets stuck when initializing the NCCL communicator.
6 +# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce`
7 +# contains many other potential cuda APIs, that are not allowed during
8 +# capturing the CUDA graph. For further details, please check
9 +# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ .
10 +#
11 +# Another rejected idea is to write a C/C++ binding for NCCL. It is usually
12 +# doable, but we often encounter issues related with nccl versions, and need
13 +# to switch between different versions of NCCL. See
14 +# https://github.com/NVIDIA/nccl/issues/1234 for more details.
15 +# A C/C++ binding is not flexible enough to handle this. It requires
16 +# recompilation of the code every time we want to switch between different
17 +# versions. This current implementation, with a **pure** Python wrapper, is
18 +# more flexible. We can easily switch between different versions of NCCL by
19 +# changing the environment variable `VLLM_NCCL_SO_PATH`, or the `so_file`
20 +# variable in the code.
21 +
22 +import ctypes
23 +import platform
24 +from dataclasses import dataclass
25 +from typing import Any, Dict, List, Optional
26 +
27 +import torch
28 +from torch.distributed import ReduceOp
29 +
30 +from vllm.logger import init_logger
31 +from vllm.utils import find_nccl_library, nccl_integrity_check
32 +
33 +logger = init_logger(__name__)
34 +
35 +# === export types and functions from nccl to Python ===
36 +# for the original nccl definition, please check
37 +# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in
38 +
39 +ncclResult_t = ctypes.c_int
40 +ncclComm_t = ctypes.c_void_p
41 +
42 +
43 +class ncclUniqueId(ctypes.Structure):
44 + _fields_ = [("internal", ctypes.c_byte * 128)]
45 +
46 +
47 +cudaStream_t = ctypes.c_void_p
48 +buffer_type = ctypes.c_void_p
49 +
50 +ncclDataType_t = ctypes.c_int
51 +
52 +
53 +class ncclDataTypeEnum:
54 + ncclInt8 = 0
55 + ncclChar = 0
56 + ncclUint8 = 1
57 + ncclInt32 = 2
58 + ncclInt = 2
59 + ncclUint32 = 3
60 + ncclInt64 = 4
61 + ncclUint64 = 5
62 + ncclFloat16 = 6
63 + ncclHalf = 6
64 + ncclFloat32 = 7
65 + ncclFloat = 7
66 + ncclFloat64 = 8
67 + ncclDouble = 8
68 + ncclBfloat16 = 9
69 + ncclNumTypes = 10
70 +
71 + @classmethod
72 + def from_torch(cls, dtype: torch.dtype) -> int:
73 + if dtype == torch.int8:
74 + return cls.ncclInt8
75 + if dtype == torch.uint8:
76 + return cls.ncclUint8
77 + if dtype == torch.int32:
78 + return cls.ncclInt32
79 + if dtype == torch.int64:
80 + return cls.ncclInt64
81 + if dtype == torch.float16:
82 + return cls.ncclFloat16
83 + if dtype == torch.float32:
84 + return cls.ncclFloat32
85 + if dtype == torch.float64:
86 + return cls.ncclFloat64
87 + if dtype == torch.bfloat16:
88 + return cls.ncclBfloat16
89 + raise ValueError(f"Unsupported dtype: {dtype}")
90 +
91 +
92 +ncclRedOp_t = ctypes.c_int
93 +
94 +
95 +class ncclRedOpTypeEnum:
96 + ncclSum = 0
97 + ncclProd = 1
98 + ncclMax = 2
99 + ncclMin = 3
100 + ncclAvg = 4
101 + ncclNumOps = 5
102 +
103 + @classmethod
104 + def from_torch(cls, op: ReduceOp) -> int:
105 + if op == ReduceOp.SUM:
106 + return cls.ncclSum
107 + if op == ReduceOp.PRODUCT:
108 + return cls.ncclProd
109 + if op == ReduceOp.MAX:
110 + return cls.ncclMax
111 + if op == ReduceOp.MIN:
112 + return cls.ncclMin
113 + if op == ReduceOp.AVG:
114 + return cls.ncclAvg
115 + raise ValueError(f"Unsupported op: {op}")
116 +
117 +
118 +@dataclass
119 +class Function:
120 + name: str
121 + restype: Any
122 + argtypes: List[Any]
123 +
124 +
125 +class NCCLLibrary:
126 + exported_functions = [
127 + # const char* ncclGetErrorString(ncclResult_t result)
128 + Function("ncclGetErrorString", ctypes.c_char_p, [ncclResult_t]),
129 + # ncclResult_t ncclGetVersion(int *version);
130 + Function("ncclGetVersion", ncclResult_t,
131 + [ctypes.POINTER(ctypes.c_int)]),
132 + # ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId);
133 + Function("ncclGetUniqueId", ncclResult_t,
134 + [ctypes.POINTER(ncclUniqueId)]),
135 + # ncclResult_t ncclCommInitRank(
136 + # ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank);
137 + # note that ncclComm_t is a pointer type, so the first argument
138 + # is a pointer to a pointer
139 + Function("ncclCommInitRank", ncclResult_t, [
140 + ctypes.POINTER(ncclComm_t), ctypes.c_int, ncclUniqueId,
141 + ctypes.c_int
142 + ]),
143 + # ncclResult_t ncclAllReduce(
144 + # const void* sendbuff, void* recvbuff, size_t count,
145 + # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm,
146 + # cudaStream_t stream);
147 + # note that cudaStream_t is a pointer type, so the last argument
148 + # is a pointer
149 + Function("ncclAllReduce", ncclResult_t, [
150 + buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
151 + ncclRedOp_t, ncclComm_t, cudaStream_t
152 + ]),
153 +
154 + # be cautious! this is a collective call, it will block until all
155 + # processes in the communicator have called this function.
156 + # because Python object destruction can happen in random order,
157 + # it is better not to call it at all.
158 + # ncclResult_t ncclCommDestroy(ncclComm_t comm);
159 + Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]),
160 + ]
161 +
162 + # class attribute to store the mapping from the path to the library
163 + # to avoid loading the same library multiple times
164 + path_to_library_cache: Dict[str, Any] = {}
165 +
166 + # class attribute to store the mapping from library path
167 + # to the corresponding dictionary
168 + path_to_dict_mapping: Dict[str, Dict[str, Any]] = {}
169 +
170 + def __init__(self, so_file: Optional[str] = None):
171 +
172 + so_file = so_file or find_nccl_library()
173 +
174 + try:
175 + # load the library in another process.
176 + # if it core dumps, it will not crash the current process
177 + nccl_integrity_check(so_file)
178 + except Exception as e:
179 + logger.error(
180 + "Failed to load NCCL library from %s ."
181 + "It is expected if you are not running on NVIDIA/AMD GPUs."
182 + "Otherwise, the nccl library might not exist, be corrupted "
183 + "or it does not support the current platform %s."
184 + "One solution is to download libnccl2 version 2.18 from "
185 + "https://developer.download.nvidia.com/compute/cuda/repos/ "
186 + "and extract the libnccl.so.2 file. If you already have the "
187 + "library, please set the environment variable VLLM_NCCL_SO_PATH"
188 + " to point to the correct nccl library path.", so_file,
189 + platform.platform())
190 + raise e
191 +
192 + if so_file not in NCCLLibrary.path_to_dict_mapping:
193 + lib = ctypes.CDLL(so_file)
194 + NCCLLibrary.path_to_library_cache[so_file] = lib
195 + self.lib = NCCLLibrary.path_to_library_cache[so_file]
196 +
197 + if so_file not in NCCLLibrary.path_to_dict_mapping:
198 + _funcs = {}
199 + for func in NCCLLibrary.exported_functions:
200 + f = getattr(self.lib, func.name)
201 + f.restype = func.restype
202 + f.argtypes = func.argtypes
203 + _funcs[func.name] = f
204 + NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
205 + self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
206 +
207 + def ncclGetErrorString(self, result: ncclResult_t) -> str:
208 + return self._funcs["ncclGetErrorString"](result).decode("utf-8")
209 +
210 + def NCCL_CHECK(self, result: ncclResult_t) -> None:
211 + if result != 0:
212 + error_str = self.ncclGetErrorString(result)
213 + raise RuntimeError(f"NCCL error: {error_str}")
214 +
215 + def ncclGetVersion(self) -> str:
216 + version = ctypes.c_int()
217 + self.NCCL_CHECK(self._funcs["ncclGetVersion"](ctypes.byref(version)))
218 + version_str = str(version.value)
219 + # something like 21903 --> "2.19.3"
220 + major = version_str[0].lstrip("0")
221 + minor = version_str[1:3].lstrip("0")
222 + patch = version_str[3:].lstrip("0")
223 + return f"{major}.{minor}.{patch}"
224 +
225 + def ncclGetUniqueId(self) -> ncclUniqueId:
226 + unique_id = ncclUniqueId()
227 + self.NCCL_CHECK(self._funcs["ncclGetUniqueId"](
228 + ctypes.byref(unique_id)))
229 + return unique_id
230 +
231 + def ncclCommInitRank(self, world_size: int, unique_id: ncclUniqueId,
232 + rank: int) -> ncclComm_t:
233 + comm = ncclComm_t()
234 + self.NCCL_CHECK(self._funcs["ncclCommInitRank"](ctypes.byref(comm),
235 + world_size, unique_id,
236 + rank))
237 + return comm
238 +
239 + def ncclAllReduce(self, sendbuff: buffer_type, recvbuff: buffer_type,
240 + count: int, datatype: int, op: int, comm: ncclComm_t,
241 + stream: cudaStream_t) -> None:
242 + # `datatype` actually should be `ncclDataType_t`
243 + # and `op` should be `ncclRedOp_t`
244 + # both are aliases of `ctypes.c_int`
245 + # when we pass int to a function, it will be converted to `ctypes.c_int`
246 + # by ctypes automatically
247 + self.NCCL_CHECK(self._funcs["ncclAllReduce"](sendbuff, recvbuff, count,
248 + datatype, op, comm,
249 + stream))
250 +
251 + def ncclCommDestroy(self, comm: ncclComm_t) -> None:
252 + self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
253 +
254 +
255 +__all__ = [
256 + "NCCLLibrary", "ncclDataTypeEnum", "ncclRedOpTypeEnum", "ncclUniqueId",
257 + "ncclComm_t", "cudaStream_t", "buffer_type"
258 +]
vllm/distributed/parallel_state.py
+60 -71
@@ -3,10 +3,10 @@
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 """Tensor and pipeline parallel groups."""
6 -import contextlib
7 -from typing import Optional
6 +from typing import List, Optional
7
8 import torch
9 +from torch.distributed import ProcessGroup
10
11 import vllm.envs as envs
12 from vllm.logger import init_logger
@@ -14,10 +14,11 @@ from vllm.logger import init_logger
14 logger = init_logger(__name__)
15
16 # Tensor model parallel group that the current rank belongs to.
17 -_TP_DEVICE_GROUP = None
18 -_TP_CPU_GROUP = None
17 +_TP_DEVICE_GROUP: Optional[ProcessGroup] = None
18 +_TP_CPU_GROUP: Optional[ProcessGroup] = None
19 +_TP_PYNCCL_COMMUNICATOR = None
20 # Pipeline model parallel group that the current rank belongs to.
20 -_PIPELINE_MODEL_PARALLEL_GROUP = None
21 +_PP_DEVICE_GROUP: Optional[ProcessGroup] = None
22
23 # when people blindly call `torch.distributed.all_reduce` etc,
24 # it will use this group. It is initialized with the `backend`
@@ -41,11 +42,16 @@ _CPU_WORLD_GROUP = None
42
43 # A list of global ranks for each pipeline group to ease calculation of the
44 # source rank when broadcasting from the first or last pipeline stage.
44 -_PIPELINE_GLOBAL_RANKS = None
45 +_PP_GLOBAL_RANKS: Optional[List[int]] = None
46
47 _LOCAL_RANK = -1
48
49
50 +def get_tp_pynccl_communicator():
51 + global _TP_PYNCCL_COMMUNICATOR
52 + return _TP_PYNCCL_COMMUNICATOR
53 +
54 +
55 def get_local_rank():
56 global _LOCAL_RANK
57 return _LOCAL_RANK
@@ -80,10 +86,20 @@ def init_distributed_environment(
86 # set the local rank
87 # local_rank is not available in torch ProcessGroup,
88 # see https://github.com/pytorch/pytorch/issues/122816
83 - if local_rank == -1 and distributed_init_method == "env://":
84 - local_rank = envs.LOCAL_RANK
89 + if local_rank == -1:
90 + # local rank not set, this usually happens in single-node
91 + # setting, where we can use rank as local rank
92 + if distributed_init_method == "env://":
93 + local_rank = envs.LOCAL_RANK
94 + else:
95 + local_rank = rank
96 global _LOCAL_RANK
97 _LOCAL_RANK = local_rank
98 + # A small all_reduce for warmup.
99 + data = torch.zeros(1)
100 + if torch.cuda.is_available():
101 + data = data.to(device=f"cuda:{local_rank}")
102 + torch.distributed.all_reduce(data)
103
104
105 def initialize_model_parallel(
@@ -133,29 +149,36 @@ def initialize_model_parallel(
149 rank = torch.distributed.get_rank()
150
151 # Build the tensor model-parallel groups.
136 - global _TP_DEVICE_GROUP, _TP_CPU_GROUP
152 + global _TP_DEVICE_GROUP, _TP_CPU_GROUP, _TP_PYNCCL_COMMUNICATOR
153 assert _TP_DEVICE_GROUP is None, (
154 "tensor model parallel group is already initialized")
155 for i in range(num_tensor_model_parallel_groups):
140 - ranks = range(i * tensor_model_parallel_size,
141 - (i + 1) * tensor_model_parallel_size)
156 + ranks = list(
157 + range(i * tensor_model_parallel_size,
158 + (i + 1) * tensor_model_parallel_size))
159 group = torch.distributed.new_group(ranks, backend=backend)
160 cpu_group = torch.distributed.new_group(ranks, backend="gloo")
161 if rank in ranks:
162 _TP_DEVICE_GROUP = group
163 _TP_CPU_GROUP = cpu_group
164
165 + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
166 + _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
167 + group=_TP_CPU_GROUP,
168 + device=_LOCAL_RANK,
169 + )
170 +
171 # Build the pipeline model-parallel groups.
149 - global _PIPELINE_MODEL_PARALLEL_GROUP
150 - global _PIPELINE_GLOBAL_RANKS
151 - assert _PIPELINE_MODEL_PARALLEL_GROUP is None, (
172 + global _PP_DEVICE_GROUP
173 + global _PP_GLOBAL_RANKS
174 + assert _PP_DEVICE_GROUP is None, (
175 "pipeline model parallel group is already initialized")
176 for i in range(num_pipeline_model_parallel_groups):
154 - ranks = range(i, world_size, num_pipeline_model_parallel_groups)
177 + ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
178 group = torch.distributed.new_group(ranks, backend=backend)
179 if rank in ranks:
157 - _PIPELINE_MODEL_PARALLEL_GROUP = group
158 - _PIPELINE_GLOBAL_RANKS = ranks
180 + _PP_DEVICE_GROUP = group
181 + _PP_GLOBAL_RANKS = ranks
182
183
184 def ensure_model_parallel_initialized(
@@ -188,8 +211,7 @@ def ensure_model_parallel_initialized(
211
212 def model_parallel_is_initialized():
213 """Check if tensor and pipeline parallel groups are initialized."""
191 - return (_TP_DEVICE_GROUP is not None
192 - and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
214 + return (_TP_DEVICE_GROUP is not None and _PP_DEVICE_GROUP is not None)
215
216
217 def get_cpu_world_group():
@@ -214,9 +236,9 @@ def get_tensor_model_parallel_cpu_group():
236
237 def get_pipeline_model_parallel_group():
238 """Get the pipeline model parallel group the caller rank belongs to."""
217 - assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, (
239 + assert _PP_DEVICE_GROUP is not None, (
240 "pipeline model parallel group is not initialized")
219 - return _PIPELINE_MODEL_PARALLEL_GROUP
241 + return _PP_DEVICE_GROUP
242
243
244 def get_tensor_model_parallel_world_size():
@@ -253,36 +275,36 @@ def get_tensor_model_parallel_src_rank():
275 def get_pipeline_model_parallel_first_rank():
276 """Return the global rank of the first process in the pipeline for the
277 current tensor parallel group"""
256 - assert _PIPELINE_GLOBAL_RANKS is not None, (
278 + assert _PP_GLOBAL_RANKS is not None, (
279 "Pipeline parallel group is not initialized")
258 - return _PIPELINE_GLOBAL_RANKS[0]
280 + return _PP_GLOBAL_RANKS[0]
281
282
283 def get_pipeline_model_parallel_last_rank():
284 """Return the global rank of the last process in the pipeline for the
285 current tensor parallel group"""
264 - assert _PIPELINE_GLOBAL_RANKS is not None, (
286 + assert _PP_GLOBAL_RANKS is not None, (
287 "Pipeline parallel group is not initialized")
288 last_rank_local = get_pipeline_model_parallel_world_size() - 1
267 - return _PIPELINE_GLOBAL_RANKS[last_rank_local]
289 + return _PP_GLOBAL_RANKS[last_rank_local]
290
291
292 def get_pipeline_model_parallel_next_rank():
293 """Return the global rank that follows the caller in the pipeline"""
272 - assert _PIPELINE_GLOBAL_RANKS is not None, (
294 + assert _PP_GLOBAL_RANKS is not None, (
295 "Pipeline parallel group is not initialized")
296 rank_in_pipeline = get_pipeline_model_parallel_rank()
297 world_size = get_pipeline_model_parallel_world_size()
276 - return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
298 + return _PP_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
299
300
301 def get_pipeline_model_parallel_prev_rank():
302 """Return the global rank that precedes the caller in the pipeline"""
281 - assert _PIPELINE_GLOBAL_RANKS is not None, (
303 + assert _PP_GLOBAL_RANKS is not None, (
304 "Pipeline parallel group is not initialized")
305 rank_in_pipeline = get_pipeline_model_parallel_rank()
306 world_size = get_pipeline_model_parallel_world_size()
285 - return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
307 + return _PP_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
308
309
310 def destroy_model_parallel():
@@ -295,45 +317,12 @@ def destroy_model_parallel():
317 if _TP_CPU_GROUP:
318 torch.distributed.destroy_process_group(_TP_CPU_GROUP)
319 _TP_CPU_GROUP = None
298 - global _PIPELINE_MODEL_PARALLEL_GROUP
299 - if _PIPELINE_MODEL_PARALLEL_GROUP:
300 - torch.distributed.destroy_process_group(_PIPELINE_MODEL_PARALLEL_GROUP)
301 - _PIPELINE_MODEL_PARALLEL_GROUP = None
302 - global _PIPELINE_GLOBAL_RANKS
303 - _PIPELINE_GLOBAL_RANKS = None
304 - from vllm.distributed.device_communicators import pynccl_utils
305 -
306 - # Destroy the pynccl states if any.
307 - pynccl_utils.destroy_process_group()
308 -
309 -
310 -# Whether to use pynccl for nccl all reduce.
311 -# We use pynccl for all reduce when using CUDA graph, because torch.distributed
312 -# is not well supported by CUDA graph.
313 -_ENABLE_PYNCCL_FOR_ALL_REDUCE = False
314 -
315 -
316 -@contextlib.contextmanager
317 -def with_pynccl_for_all_reduce():
318 - from vllm.distributed.device_communicators import pynccl_utils
319 - """use pynccl instead of torch.distributed for all reduce"""
320 - tp_size = get_tensor_model_parallel_world_size()
321 - if tp_size == 1:
322 - # No-op.
323 - # NOTE(woosuk): We don't initialize pynccl when tp_size is 1.
324 - yield
325 - else:
326 - global _ENABLE_PYNCCL_FOR_ALL_REDUCE
327 - old = _ENABLE_PYNCCL_FOR_ALL_REDUCE
328 - _ENABLE_PYNCCL_FOR_ALL_REDUCE = True
329 -
330 - stream = torch.cuda.current_stream()
331 - with pynccl_utils.set_pynccl_stream(stream):
332 - yield
333 - _ENABLE_PYNCCL_FOR_ALL_REDUCE = old
334 -
335 -
336 -def is_pynccl_enabled_for_all_reduce():
337 - """check if pynccl is enabled for all reduce"""
338 - global _ENABLE_PYNCCL_FOR_ALL_REDUCE
339 - return _ENABLE_PYNCCL_FOR_ALL_REDUCE
320 + global _TP_PYNCCL_COMMUNICATOR
321 + _TP_PYNCCL_COMMUNICATOR = None
322 +
323 + global _PP_DEVICE_GROUP
324 + if _PP_DEVICE_GROUP:
325 + torch.distributed.destroy_process_group(_PP_DEVICE_GROUP)
326 + _PP_DEVICE_GROUP = None
327 + global _PP_GLOBAL_RANKS
328 + _PP_GLOBAL_RANKS = None
vllm/worker/model_runner.py
+5 -20
@@ -1,4 +1,3 @@
1 -import contextlib
1 import time
2 from enum import IntEnum
3 from typing import Dict, List, NamedTuple, Optional, Set, Tuple
@@ -12,9 +11,9 @@ from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 ModelConfig, ParallelConfig, SchedulerConfig,
13 VisionLanguageConfig)
15 -from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
16 -from vllm.distributed.device_communicators import (custom_all_reduce,
17 - pynccl_utils)
14 +from vllm.distributed import broadcast_tensor_dict
15 +from vllm.distributed.communication_op import graph_capture_mode
16 +from vllm.distributed.device_communicators import custom_all_reduce
17 from vllm.logger import init_logger
18 from vllm.lora.layers import LoRAMapping
19 from vllm.lora.request import LoRARequest
@@ -917,10 +916,6 @@ class ModelRunner:
916 Since it is used for decoding-only, it assumes there's only 1 token
917 per sequence in the batch.
918 """
920 - # NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
921 - # deleted before the CUDA graphs.
922 - self.pynccl_backend = pynccl_utils.get_nccl_backend()
923 -
919 assert not self.model_config.enforce_eager
920 logger.info("Capturing the model for CUDA graphs. This may lead to "
921 "unexpected consequences if the model is not static. To "
@@ -1046,7 +1041,7 @@ class CUDAGraphRunner:
1041 # Run the model once without capturing the graph.
1042 # This is to make sure that the captured graph does not include the
1043 # kernel launches for initial benchmarking (e.g., Triton autotune).
1049 - with _maybe_pynccl():
1044 + with graph_capture_mode():
1045 self.model(
1046 input_ids,
1047 positions,
@@ -1061,7 +1056,7 @@ class CUDAGraphRunner:
1056 # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
1057 self._graph = torch.cuda.CUDAGraph()
1058 with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
1064 - with _maybe_pynccl():
1059 + with graph_capture_mode():
1060 hidden_states = self.model(
1061 input_ids,
1062 positions,
@@ -1113,16 +1108,6 @@ class CUDAGraphRunner:
1108 return self.forward(*args, **kwargs)
1109
1110
1116 -@contextlib.contextmanager
1117 -def _maybe_pynccl():
1118 - if pynccl_utils.is_initialized(
1119 - ) and not custom_all_reduce.is_initialized():
1120 - with with_pynccl_for_all_reduce():
1121 - yield
1122 - else:
1123 - yield
1124 -
1125 -
1111 def _get_graph_batch_size(batch_size: int) -> int:
1112 """Returns the padded batch size given actual batch size.
1113
vllm/worker/worker.py
-21
@@ -11,9 +11,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 VisionLanguageConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
14 - get_tensor_model_parallel_cpu_group,
14 init_distributed_environment)
16 -from vllm.distributed.device_communicators import pynccl_utils
15 from vllm.distributed.device_communicators.custom_all_reduce import (
16 init_custom_ar)
17 from vllm.lora.request import LoRARequest
@@ -306,29 +304,10 @@ def init_worker_distributed_environment(
304 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
305 parallel_config.pipeline_parallel_size)
306
309 - if pynccl_utils.is_initialized():
310 - pynccl_world_size = pynccl_utils.get_world_size()
311 - if pynccl_world_size != parallel_config.world_size:
312 - raise RuntimeError(
313 - "pynccl is already initialized but the pynccl world "
314 - "size does not match parallel_config.world_size "
315 - f"({pynccl_world_size} vs. {parallel_config.world_size}).")
316 - elif parallel_config.world_size > 1:
317 - # NOTE(woosuk): We don't initialize pynccl process group when world size
318 - # is 1.
319 - # NOTE(kaichao): By default, pynccl is initialized for tp group.
320 - pynccl_utils.init_process_group(
321 - group=get_tensor_model_parallel_cpu_group())
322 -
307 # Initialize a custom fast all-reduce implementation.
308 if not parallel_config.disable_custom_all_reduce:
309 init_custom_ar()
310
327 - # A small all_reduce for warmup.
328 - torch.distributed.all_reduce(torch.zeros(1).cuda())
329 - if pynccl_utils.is_initialized():
330 - pynccl_utils.all_reduce(torch.zeros(1).cuda())
331 -
311
312 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
313 # Check if the GPU supports the dtype.