@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6dc1fc9cfe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] nccl integrity check and test (#4155)

[Core] Add integrity check during initialization; add test for it (#4155)

youkaichao committed Apr 17, 2024 at 22:28 UTC 6dc1fc9cfed74f63f3e90c60e4d6ad7065bd4529
4 files changed +107 -26
.buildkite/test-pipeline.yaml
+1
@@ -33,6 +33,7 @@ steps:
33 num_gpus: 2 # only support 1 or 2 for now.
34 commands:
35 - pytest -v -s test_pynccl.py
36 + - pytest -v -s test_pynccl_library.py
37 - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
38 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_basic_distributed_correctness.py
39 - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_chunked_prefill_distributed.py
tests/distributed/test_pynccl_library.py new
+43
@@ -0,0 +1,43 @@
1 +import multiprocessing
2 +import tempfile
3 +
4 +
5 +def target_fn(env, filepath):
6 + from vllm.utils import update_environment_variables
7 + update_environment_variables(env)
8 + from vllm.utils import nccl_integrity_check
9 + nccl_integrity_check(filepath)
10 +
11 +
12 +def test_library_file():
13 + # note: don't import vllm.distributed.device_communicators.pynccl
14 + # before running this test, otherwise the library file will be loaded
15 + # and it might interfere with the test
16 + from vllm.utils import find_nccl_library
17 + so_file = find_nccl_library()
18 + with open(so_file, 'rb') as f:
19 + content = f.read()
20 + try:
21 + # corrupt the library file, should raise an exception
22 + with open(so_file, 'wb') as f:
23 + f.write(content[:len(content) // 2])
24 + p = multiprocessing.Process(target=target_fn, args=({}, so_file))
25 + p.start()
26 + p.join()
27 + assert p.exitcode != 0
28 +
29 + # move the library file to a tmp path
30 + # test VLLM_NCCL_SO_PATH
31 + fd, path = tempfile.mkstemp()
32 + with open(path, 'wb') as f:
33 + f.write(content)
34 + p = multiprocessing.Process(target=target_fn,
35 + args=({
36 + "VLLM_NCCL_SO_PATH": path
37 + }, path))
38 + p.start()
39 + p.join()
40 + assert p.exitcode == 0
41 + finally:
42 + with open(so_file, 'wb') as f:
43 + f.write(content)
vllm/distributed/device_communicators/pynccl.py
+12 -26
@@ -21,8 +21,7 @@
21
22 import ctypes
23 import datetime
24 -import glob
25 -import os
24 +import platform
25
26 # ===================== import region =====================
27 import torch
@@ -30,40 +29,27 @@ import torch.distributed as dist
29 from torch.distributed import ReduceOp
30
31 from vllm.logger import init_logger
32 +from vllm.utils import find_nccl_library, nccl_integrity_check
33
34 logger = init_logger(__name__)
35
36 -so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
37 -
38 -# check if we have vllm-managed nccl
39 -vllm_nccl_path = None
40 -if torch.version.cuda is not None:
41 - cuda_major = torch.version.cuda.split(".")[0]
42 - path = os.path.expanduser(
43 - f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
44 - files = glob.glob(path)
45 - vllm_nccl_path = files[0] if files else None
46 -
47 -# manually load the nccl library
48 -if so_file:
49 - logger.info(
50 - f"Loading nccl from environment variable VLLM_NCCL_SO_PATH={so_file}")
51 -else:
52 - if torch.version.cuda is not None:
53 - so_file = vllm_nccl_path or "libnccl.so.2"
54 - elif torch.version.hip is not None:
55 - so_file = "librccl.so.1"
56 - else:
57 - raise ValueError("NCCL only supports CUDA and ROCm backends.")
58 - logger.info(f"Loading nccl from library {so_file}")
36 +so_file = find_nccl_library()
37
38 try:
39 + # load the library in another process.
40 + # if it core dumps, it will not crash the current process
41 + nccl_integrity_check(so_file)
42 nccl = ctypes.CDLL(so_file)
43 except Exception as e:
44 logger.error(
45 f"Failed to load NCCL library from {so_file} ."
46 "It is expected if you are not running on NVIDIA/AMD GPUs."
66 - "Otherwise please set the environment variable VLLM_NCCL_SO_PATH"
47 + "Otherwise, the nccl library might not exist, be corrupted "
48 + f"or it does not support the current platform {platform.platform()}."
49 + f"One solution is to download libnccl2 version 2.18 from "
50 + f"https://developer.download.nvidia.com/compute/cuda/repos/ "
51 + f"and extract the libnccl.so.2 file. If you already have the "
52 + f"library, please set the environment variable VLLM_NCCL_SO_PATH"
53 " to point to the correct nccl library path.")
54 raise e
55
vllm/utils.py
+51
@@ -1,6 +1,7 @@
1 import asyncio
2 import enum
3 import gc
4 +import glob
5 import os
6 import socket
7 import subprocess
@@ -517,3 +518,53 @@ def init_cached_hf_modules():
518 """
519 from transformers.dynamic_module_utils import init_hf_modules
520 init_hf_modules()
521 +
522 +
523 +def nccl_integrity_check(filepath):
524 + """
525 + when the library is corrupted, we cannot catch
526 + the exception in python. it will crash the process.
527 + instead, we use the exit code of `ldd` to check
528 + if the library is corrupted. if not, we will return
529 + the version of the library.
530 + """
531 + exit_code = os.system(f"ldd {filepath} 2>&1 > /dev/null")
532 + if exit_code != 0:
533 + raise RuntimeError(f"Failed to load NCCL library from {filepath} .")
534 + import ctypes
535 +
536 + nccl = ctypes.CDLL(filepath)
537 + version = ctypes.c_int()
538 + nccl.ncclGetVersion.restype = ctypes.c_int
539 + nccl.ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
540 + result = nccl.ncclGetVersion(ctypes.byref(version))
541 + assert result == 0
542 + return version.value
543 +
544 +
545 +def find_nccl_library():
546 + so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
547 +
548 + # check if we have vllm-managed nccl
549 + vllm_nccl_path = None
550 + if torch.version.cuda is not None:
551 + cuda_major = torch.version.cuda.split(".")[0]
552 + path = os.path.expanduser(
553 + f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
554 + files = glob.glob(path)
555 + vllm_nccl_path = files[0] if files else None
556 +
557 + # manually load the nccl library
558 + if so_file:
559 + logger.info(
560 + f"Found nccl from environment variable VLLM_NCCL_SO_PATH={so_file}"
561 + )
562 + else:
563 + if torch.version.cuda is not None:
564 + so_file = vllm_nccl_path or "libnccl.so.2"
565 + elif torch.version.hip is not None:
566 + so_file = "librccl.so.1"
567 + else:
568 + raise ValueError("NCCL only supports CUDA and ROCm backends.")
569 + logger.info(f"Found nccl from library {so_file}")
570 + return so_file