A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] nccl integrity check and test (#4155)
[Core] Add integrity check during initialization; add test for it (#4155)
youkaichao committed
Apr 17, 2024 at 22:28 UTC
6dc1fc9cfed74f63f3e90c60e4d6ad7065bd4529
4 files changed
+107
-26
.buildkite/test-pipeline.yaml
+1
@@ -33,6 +33,7 @@ steps:
33
num_gpus: 2 # only support 1 or 2 for now.
34
commands:
35
- pytest -v -s test_pynccl.py
36
+ - pytest -v -s test_pynccl_library.py
37
- TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
38
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_basic_distributed_correctness.py
39
- TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_chunked_prefill_distributed.py
tests/distributed/test_pynccl_library.py
new
+43
@@ -0,0 +1,43 @@
1
+import multiprocessing
2
+import tempfile
3
+
4
+
5
+def target_fn(env, filepath):
6
+ from vllm.utils import update_environment_variables
7
+ update_environment_variables(env)
8
+ from vllm.utils import nccl_integrity_check
9
+ nccl_integrity_check(filepath)
10
+
11
+
12
+def test_library_file():
13
+ # note: don't import vllm.distributed.device_communicators.pynccl
14
+ # before running this test, otherwise the library file will be loaded
15
+ # and it might interfere with the test
16
+ from vllm.utils import find_nccl_library
17
+ so_file = find_nccl_library()
18
+ with open(so_file, 'rb') as f:
19
+ content = f.read()
20
+ try:
21
+ # corrupt the library file, should raise an exception
22
+ with open(so_file, 'wb') as f:
23
+ f.write(content[:len(content) // 2])
24
+ p = multiprocessing.Process(target=target_fn, args=({}, so_file))
25
+ p.start()
26
+ p.join()
27
+ assert p.exitcode != 0
28
+
29
+ # move the library file to a tmp path
30
+ # test VLLM_NCCL_SO_PATH
31
+ fd, path = tempfile.mkstemp()
32
+ with open(path, 'wb') as f:
33
+ f.write(content)
34
+ p = multiprocessing.Process(target=target_fn,
35
+ args=({
36
+ "VLLM_NCCL_SO_PATH": path
37
+ }, path))
38
+ p.start()
39
+ p.join()
40
+ assert p.exitcode == 0
41
+ finally:
42
+ with open(so_file, 'wb') as f:
43
+ f.write(content)
vllm/distributed/device_communicators/pynccl.py
+12
-26
@@ -21,8 +21,7 @@
21
22
import ctypes
23
import datetime
24
-import glob
25
-import os
24
+import platform
25
26
# ===================== import region =====================
27
import torch
@@ -30,40 +29,27 @@ import torch.distributed as dist
29
from torch.distributed import ReduceOp
30
31
from vllm.logger import init_logger
32
+from vllm.utils import find_nccl_library, nccl_integrity_check
33
34
logger = init_logger(__name__)
35
36
-so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
37
-
38
-# check if we have vllm-managed nccl
39
-vllm_nccl_path = None
40
-if torch.version.cuda is not None:
41
- cuda_major = torch.version.cuda.split(".")[0]
42
- path = os.path.expanduser(
43
- f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
44
- files = glob.glob(path)
45
- vllm_nccl_path = files[0] if files else None
46
-
47
-# manually load the nccl library
48
-if so_file:
49
- logger.info(
50
- f"Loading nccl from environment variable VLLM_NCCL_SO_PATH={so_file}")
51
-else:
52
- if torch.version.cuda is not None:
53
- so_file = vllm_nccl_path or "libnccl.so.2"
54
- elif torch.version.hip is not None:
55
- so_file = "librccl.so.1"
56
- else:
57
- raise ValueError("NCCL only supports CUDA and ROCm backends.")
58
- logger.info(f"Loading nccl from library {so_file}")
36
+so_file = find_nccl_library()
37
38
try:
39
+ # load the library in another process.
40
+ # if it core dumps, it will not crash the current process
41
+ nccl_integrity_check(so_file)
42
nccl = ctypes.CDLL(so_file)
43
except Exception as e:
44
logger.error(
45
f"Failed to load NCCL library from {so_file} ."
46
"It is expected if you are not running on NVIDIA/AMD GPUs."
66
- "Otherwise please set the environment variable VLLM_NCCL_SO_PATH"
47
+ "Otherwise, the nccl library might not exist, be corrupted "
48
+ f"or it does not support the current platform {platform.platform()}."
49
+ f"One solution is to download libnccl2 version 2.18 from "
50
+ f"https://developer.download.nvidia.com/compute/cuda/repos/ "
51
+ f"and extract the libnccl.so.2 file. If you already have the "
52
+ f"library, please set the environment variable VLLM_NCCL_SO_PATH"
53
" to point to the correct nccl library path.")
54
raise e
55
vllm/utils.py
+51
@@ -1,6 +1,7 @@
1
import asyncio
2
import enum
3
import gc
4
+import glob
5
import os
6
import socket
7
import subprocess
@@ -517,3 +518,53 @@ def init_cached_hf_modules():
518
"""
519
from transformers.dynamic_module_utils import init_hf_modules
520
init_hf_modules()
521
+
522
+
523
+def nccl_integrity_check(filepath):
524
+ """
525
+ when the library is corrupted, we cannot catch
526
+ the exception in python. it will crash the process.
527
+ instead, we use the exit code of `ldd` to check
528
+ if the library is corrupted. if not, we will return
529
+ the version of the library.
530
+ """
531
+ exit_code = os.system(f"ldd {filepath} 2>&1 > /dev/null")
532
+ if exit_code != 0:
533
+ raise RuntimeError(f"Failed to load NCCL library from {filepath} .")
534
+ import ctypes
535
+
536
+ nccl = ctypes.CDLL(filepath)
537
+ version = ctypes.c_int()
538
+ nccl.ncclGetVersion.restype = ctypes.c_int
539
+ nccl.ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
540
+ result = nccl.ncclGetVersion(ctypes.byref(version))
541
+ assert result == 0
542
+ return version.value
543
+
544
+
545
+def find_nccl_library():
546
+ so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
547
+
548
+ # check if we have vllm-managed nccl
549
+ vllm_nccl_path = None
550
+ if torch.version.cuda is not None:
551
+ cuda_major = torch.version.cuda.split(".")[0]
552
+ path = os.path.expanduser(
553
+ f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
554
+ files = glob.glob(path)
555
+ vllm_nccl_path = files[0] if files else None
556
+
557
+ # manually load the nccl library
558
+ if so_file:
559
+ logger.info(
560
+ f"Found nccl from environment variable VLLM_NCCL_SO_PATH={so_file}"
561
+ )
562
+ else:
563
+ if torch.version.cuda is not None:
564
+ so_file = vllm_nccl_path or "libnccl.so.2"
565
+ elif torch.version.hip is not None:
566
+ so_file = "librccl.so.1"
567
+ else:
568
+ raise ValueError("NCCL only supports CUDA and ROCm backends.")
569
+ logger.info(f"Found nccl from library {so_file}")
570
+ return so_file