@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5bd3c65072
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core][Optimization] remove vllm-nccl (#5091)

youkaichao committed May 28, 2024 at 22:13 UTC 5bd3c650721cc5de451f034bcbed37d1a1a4116c
7 files changed +21 -100
.buildkite/test-pipeline.yaml
-1
@@ -37,7 +37,6 @@ steps:
37 working_dir: "/vllm-workspace/tests"
38 num_gpus: 2
39 commands:
40 - - pytest -v -s distributed/test_pynccl_library.py
40 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
41 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
42 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
requirements-cuda.txt
-1
@@ -4,7 +4,6 @@
4 # Dependencies for NVIDIA GPUs
5 ray >= 2.9
6 nvidia-ml-py # for pynvml package
7 -vllm-nccl-cu12>=2.18,<2.19 # for downloading nccl library
7 torch == 2.3.0
8 xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
9 vllm-flash-attn == 2.5.8.post2 # Requires PyTorch 2.3.0
setup.py
+2 -5
@@ -358,11 +358,8 @@ def get_requirements() -> List[str]:
358 cuda_major, cuda_minor = torch.version.cuda.split(".")
359 modified_requirements = []
360 for req in requirements:
361 - if "vllm-nccl-cu12" in req:
362 - req = req.replace("vllm-nccl-cu12",
363 - f"vllm-nccl-cu{cuda_major}")
364 - elif ("vllm-flash-attn" in req
365 - and not (cuda_major == "12" and cuda_minor == "1")):
361 + if ("vllm-flash-attn" in req
362 + and not (cuda_major == "12" and cuda_minor == "1")):
363 # vllm-flash-attn is built only for CUDA 12.1.
364 # Skip for other versions.
365 continue
tests/distributed/test_pynccl_library.py deleted
-43
@@ -1,43 +0,0 @@
1 -import multiprocessing
2 -import tempfile
3 -
4 -
5 -def target_fn(env, filepath):
6 - from vllm.utils import update_environment_variables
7 - update_environment_variables(env)
8 - from vllm.utils import nccl_integrity_check
9 - nccl_integrity_check(filepath)
10 -
11 -
12 -def test_library_file():
13 - # note: don't import vllm.distributed.device_communicators.pynccl
14 - # before running this test, otherwise the library file will be loaded
15 - # and it might interfere with the test
16 - from vllm.utils import find_nccl_library
17 - so_file = find_nccl_library()
18 - with open(so_file, 'rb') as f:
19 - content = f.read()
20 - try:
21 - # corrupt the library file, should raise an exception
22 - with open(so_file, 'wb') as f:
23 - f.write(content[:len(content) // 2])
24 - p = multiprocessing.Process(target=target_fn, args=({}, so_file))
25 - p.start()
26 - p.join()
27 - assert p.exitcode != 0
28 -
29 - # move the library file to a tmp path
30 - # test VLLM_NCCL_SO_PATH
31 - fd, path = tempfile.mkstemp()
32 - with open(path, 'wb') as f:
33 - f.write(content)
34 - p = multiprocessing.Process(target=target_fn,
35 - args=({
36 - "VLLM_NCCL_SO_PATH": path
37 - }, path))
38 - p.start()
39 - p.join()
40 - assert p.exitcode == 0
41 - finally:
42 - with open(so_file, 'wb') as f:
43 - f.write(content)
vllm/distributed/device_communicators/pynccl_wrapper.py
+7 -13
@@ -28,7 +28,7 @@ import torch
28 from torch.distributed import ReduceOp
29
30 from vllm.logger import init_logger
31 -from vllm.utils import find_nccl_library, nccl_integrity_check
31 +from vllm.utils import find_nccl_library
32
33 logger = init_logger(__name__)
34
@@ -188,28 +188,22 @@ class NCCLLibrary:
188 so_file = so_file or find_nccl_library()
189
190 try:
191 - # load the library in another process.
192 - # if it core dumps, it will not crash the current process
193 - nccl_integrity_check(so_file)
191 + if so_file not in NCCLLibrary.path_to_dict_mapping:
192 + lib = ctypes.CDLL(so_file)
193 + NCCLLibrary.path_to_library_cache[so_file] = lib
194 + self.lib = NCCLLibrary.path_to_library_cache[so_file]
195 except Exception as e:
196 logger.error(
197 "Failed to load NCCL library from %s ."
198 "It is expected if you are not running on NVIDIA/AMD GPUs."
199 "Otherwise, the nccl library might not exist, be corrupted "
200 "or it does not support the current platform %s."
200 - "One solution is to download libnccl2 version 2.18 from "
201 - "https://developer.download.nvidia.com/compute/cuda/repos/ "
202 - "and extract the libnccl.so.2 file. If you already have the "
203 - "library, please set the environment variable VLLM_NCCL_SO_PATH"
201 + "If you already have the library, please set the "
202 + "environment variable VLLM_NCCL_SO_PATH"
203 " to point to the correct nccl library path.", so_file,
204 platform.platform())
205 raise e
206
208 - if so_file not in NCCLLibrary.path_to_dict_mapping:
209 - lib = ctypes.CDLL(so_file)
210 - NCCLLibrary.path_to_library_cache[so_file] = lib
211 - self.lib = NCCLLibrary.path_to_library_cache[so_file]
212 -
207 if so_file not in NCCLLibrary.path_to_dict_mapping:
208 _funcs = {}
209 for func in NCCLLibrary.exported_functions:
vllm/utils.py
+8 -35
@@ -2,7 +2,6 @@ import asyncio
2 import datetime
3 import enum
4 import gc
5 -import glob
5 import os
6 import socket
7 import subprocess
@@ -565,28 +564,6 @@ def init_cached_hf_modules():
564 init_hf_modules()
565
566
568 -def nccl_integrity_check(filepath):
569 - """
570 - when the library is corrupted, we cannot catch
571 - the exception in python. it will crash the process.
572 - instead, we use the exit code of `ldd` to check
573 - if the library is corrupted. if not, we will return
574 - the version of the library.
575 - """
576 - exit_code = os.system(f"ldd {filepath} 2>&1 > /dev/null")
577 - if exit_code != 0:
578 - raise RuntimeError(f"Failed to load NCCL library from {filepath} .")
579 - import ctypes
580 -
581 - nccl = ctypes.CDLL(filepath)
582 - version = ctypes.c_int()
583 - nccl.ncclGetVersion.restype = ctypes.c_int
584 - nccl.ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
585 - result = nccl.ncclGetVersion(ctypes.byref(version))
586 - assert result == 0
587 - return version.value
588 -
589 -
567 @lru_cache(maxsize=None)
568 def find_library(lib_name: str) -> str:
569 """
@@ -616,17 +593,13 @@ def find_library(lib_name: str) -> str:
593
594
595 def find_nccl_library():
596 + """
597 + We either use the library file specified by the `VLLM_NCCL_SO_PATH`
598 + environment variable, or we find the library file brought by PyTorch.
599 + After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be
600 + found by `ctypes` automatically.
601 + """
602 so_file = envs.VLLM_NCCL_SO_PATH
620 - VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
621 -
622 - # check if we have vllm-managed nccl
623 - vllm_nccl_path = None
624 - if torch.version.cuda is not None:
625 - cuda_major = torch.version.cuda.split(".")[0]
626 - path = os.path.expanduser(
627 - f"{VLLM_CONFIG_ROOT}/vllm/nccl/cu{cuda_major}/libnccl.so.*")
628 - files = glob.glob(path)
629 - vllm_nccl_path = files[0] if files else None
603
604 # manually load the nccl library
605 if so_file:
@@ -635,9 +608,9 @@ def find_nccl_library():
608 so_file)
609 else:
610 if torch.version.cuda is not None:
638 - so_file = vllm_nccl_path or find_library("libnccl.so.2")
611 + so_file = "libnccl.so.2"
612 elif torch.version.hip is not None:
640 - so_file = find_library("librccl.so.1")
613 + so_file = "librccl.so.1"
614 else:
615 raise ValueError("NCCL only supports CUDA and ROCm backends.")
616 logger.info("Found nccl from library %s", so_file)
vllm/worker/worker_base.py
+4 -2
@@ -121,12 +121,14 @@ class WorkerWrapperBase:
121
122 def init_worker(self, *args, **kwargs):
123 """
124 - Actual initialization of the worker class, and set up
125 - function tracing if required.
124 + Here we inject some common logic before initializing the worker.
125 Arguments are passed to the worker class constructor.
126 """
127 enable_trace_function_call_for_thread()
128
129 + # see https://github.com/NVIDIA/nccl/issues/1234
130 + os.environ['NCCL_CUMEM_ENABLE'] = '0'
131 +
132 mod = importlib.import_module(self.worker_module_name)
133 worker_class = getattr(mod, self.worker_class_name)
134 self.worker = worker_class(*args, **kwargs)