A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core][Optimization] remove vllm-nccl (#5091)
youkaichao committed
May 28, 2024 at 22:13 UTC
5bd3c650721cc5de451f034bcbed37d1a1a4116c
7 files changed
+21
-100
.buildkite/test-pipeline.yaml
-1
@@ -37,7 +37,6 @@ steps:
37
working_dir: "/vllm-workspace/tests"
38
num_gpus: 2
39
commands:
40
- - pytest -v -s distributed/test_pynccl_library.py
40
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
41
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
42
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
requirements-cuda.txt
-1
@@ -4,7 +4,6 @@
4
# Dependencies for NVIDIA GPUs
5
ray >= 2.9
6
nvidia-ml-py # for pynvml package
7
-vllm-nccl-cu12>=2.18,<2.19 # for downloading nccl library
7
torch == 2.3.0
8
xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
9
vllm-flash-attn == 2.5.8.post2 # Requires PyTorch 2.3.0
setup.py
+2
-5
@@ -358,11 +358,8 @@ def get_requirements() -> List[str]:
358
cuda_major, cuda_minor = torch.version.cuda.split(".")
359
modified_requirements = []
360
for req in requirements:
361
- if "vllm-nccl-cu12" in req:
362
- req = req.replace("vllm-nccl-cu12",
363
- f"vllm-nccl-cu{cuda_major}")
364
- elif ("vllm-flash-attn" in req
365
- and not (cuda_major == "12" and cuda_minor == "1")):
361
+ if ("vllm-flash-attn" in req
362
+ and not (cuda_major == "12" and cuda_minor == "1")):
363
# vllm-flash-attn is built only for CUDA 12.1.
364
# Skip for other versions.
365
continue
tests/distributed/test_pynccl_library.py
deleted
-43
@@ -1,43 +0,0 @@
1
-import multiprocessing
2
-import tempfile
3
-
4
-
5
-def target_fn(env, filepath):
6
- from vllm.utils import update_environment_variables
7
- update_environment_variables(env)
8
- from vllm.utils import nccl_integrity_check
9
- nccl_integrity_check(filepath)
10
-
11
-
12
-def test_library_file():
13
- # note: don't import vllm.distributed.device_communicators.pynccl
14
- # before running this test, otherwise the library file will be loaded
15
- # and it might interfere with the test
16
- from vllm.utils import find_nccl_library
17
- so_file = find_nccl_library()
18
- with open(so_file, 'rb') as f:
19
- content = f.read()
20
- try:
21
- # corrupt the library file, should raise an exception
22
- with open(so_file, 'wb') as f:
23
- f.write(content[:len(content) // 2])
24
- p = multiprocessing.Process(target=target_fn, args=({}, so_file))
25
- p.start()
26
- p.join()
27
- assert p.exitcode != 0
28
-
29
- # move the library file to a tmp path
30
- # test VLLM_NCCL_SO_PATH
31
- fd, path = tempfile.mkstemp()
32
- with open(path, 'wb') as f:
33
- f.write(content)
34
- p = multiprocessing.Process(target=target_fn,
35
- args=({
36
- "VLLM_NCCL_SO_PATH": path
37
- }, path))
38
- p.start()
39
- p.join()
40
- assert p.exitcode == 0
41
- finally:
42
- with open(so_file, 'wb') as f:
43
- f.write(content)
vllm/distributed/device_communicators/pynccl_wrapper.py
+7
-13
@@ -28,7 +28,7 @@ import torch
28
from torch.distributed import ReduceOp
29
30
from vllm.logger import init_logger
31
-from vllm.utils import find_nccl_library, nccl_integrity_check
31
+from vllm.utils import find_nccl_library
32
33
logger = init_logger(__name__)
34
@@ -188,28 +188,22 @@ class NCCLLibrary:
188
so_file = so_file or find_nccl_library()
189
190
try:
191
- # load the library in another process.
192
- # if it core dumps, it will not crash the current process
193
- nccl_integrity_check(so_file)
191
+ if so_file not in NCCLLibrary.path_to_dict_mapping:
192
+ lib = ctypes.CDLL(so_file)
193
+ NCCLLibrary.path_to_library_cache[so_file] = lib
194
+ self.lib = NCCLLibrary.path_to_library_cache[so_file]
195
except Exception as e:
196
logger.error(
197
"Failed to load NCCL library from %s ."
198
"It is expected if you are not running on NVIDIA/AMD GPUs."
199
"Otherwise, the nccl library might not exist, be corrupted "
200
"or it does not support the current platform %s."
200
- "One solution is to download libnccl2 version 2.18 from "
201
- "https://developer.download.nvidia.com/compute/cuda/repos/ "
202
- "and extract the libnccl.so.2 file. If you already have the "
203
- "library, please set the environment variable VLLM_NCCL_SO_PATH"
201
+ "If you already have the library, please set the "
202
+ "environment variable VLLM_NCCL_SO_PATH"
203
" to point to the correct nccl library path.", so_file,
204
platform.platform())
205
raise e
206
208
- if so_file not in NCCLLibrary.path_to_dict_mapping:
209
- lib = ctypes.CDLL(so_file)
210
- NCCLLibrary.path_to_library_cache[so_file] = lib
211
- self.lib = NCCLLibrary.path_to_library_cache[so_file]
212
-
207
if so_file not in NCCLLibrary.path_to_dict_mapping:
208
_funcs = {}
209
for func in NCCLLibrary.exported_functions:
vllm/utils.py
+8
-35
@@ -2,7 +2,6 @@ import asyncio
2
import datetime
3
import enum
4
import gc
5
-import glob
5
import os
6
import socket
7
import subprocess
@@ -565,28 +564,6 @@ def init_cached_hf_modules():
564
init_hf_modules()
565
566
568
-def nccl_integrity_check(filepath):
569
- """
570
- when the library is corrupted, we cannot catch
571
- the exception in python. it will crash the process.
572
- instead, we use the exit code of `ldd` to check
573
- if the library is corrupted. if not, we will return
574
- the version of the library.
575
- """
576
- exit_code = os.system(f"ldd {filepath} 2>&1 > /dev/null")
577
- if exit_code != 0:
578
- raise RuntimeError(f"Failed to load NCCL library from {filepath} .")
579
- import ctypes
580
-
581
- nccl = ctypes.CDLL(filepath)
582
- version = ctypes.c_int()
583
- nccl.ncclGetVersion.restype = ctypes.c_int
584
- nccl.ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
585
- result = nccl.ncclGetVersion(ctypes.byref(version))
586
- assert result == 0
587
- return version.value
588
-
589
-
567
@lru_cache(maxsize=None)
568
def find_library(lib_name: str) -> str:
569
"""
@@ -616,17 +593,13 @@ def find_library(lib_name: str) -> str:
593
594
595
def find_nccl_library():
596
+ """
597
+ We either use the library file specified by the `VLLM_NCCL_SO_PATH`
598
+ environment variable, or we find the library file brought by PyTorch.
599
+ After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be
600
+ found by `ctypes` automatically.
601
+ """
602
so_file = envs.VLLM_NCCL_SO_PATH
620
- VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
621
-
622
- # check if we have vllm-managed nccl
623
- vllm_nccl_path = None
624
- if torch.version.cuda is not None:
625
- cuda_major = torch.version.cuda.split(".")[0]
626
- path = os.path.expanduser(
627
- f"{VLLM_CONFIG_ROOT}/vllm/nccl/cu{cuda_major}/libnccl.so.*")
628
- files = glob.glob(path)
629
- vllm_nccl_path = files[0] if files else None
603
604
# manually load the nccl library
605
if so_file:
@@ -635,9 +608,9 @@ def find_nccl_library():
608
so_file)
609
else:
610
if torch.version.cuda is not None:
638
- so_file = vllm_nccl_path or find_library("libnccl.so.2")
611
+ so_file = "libnccl.so.2"
612
elif torch.version.hip is not None:
640
- so_file = find_library("librccl.so.1")
613
+ so_file = "librccl.so.1"
614
else:
615
raise ValueError("NCCL only supports CUDA and ROCm backends.")
616
logger.info("Found nccl from library %s", so_file)
vllm/worker/worker_base.py
+4
-2
@@ -121,12 +121,14 @@ class WorkerWrapperBase:
121
122
def init_worker(self, *args, **kwargs):
123
"""
124
- Actual initialization of the worker class, and set up
125
- function tracing if required.
124
+ Here we inject some common logic before initializing the worker.
125
Arguments are passed to the worker class constructor.
126
"""
127
enable_trace_function_call_for_thread()
128
129
+ # see https://github.com/NVIDIA/nccl/issues/1234
130
+ os.environ['NCCL_CUMEM_ENABLE'] = '0'
131
+
132
mod = importlib.import_module(self.worker_module_name)
133
worker_class = getattr(mod, self.worker_class_name)
134
self.worker = worker_class(*args, **kwargs)