@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 16a1cc9bb2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[misc][distributed] improve libcudart.so finding (#7127)

youkaichao committed Aug 4, 2024 at 11:31 UTC 16a1cc9bb2b4bba82d78f329e5a89b44a5523ac8
2 files changed +25 -23
vllm/distributed/device_communicators/cuda_wrapper.py
+22 -22
@@ -4,9 +4,6 @@ convenient for use when we just need to call a few functions.
4 """
5
6 import ctypes
7 -import glob
8 -import os
9 -import sys
7 from dataclasses import dataclass
8 from typing import Any, Dict, List, Optional
9
@@ -36,24 +33,25 @@ class Function:
33 argtypes: List[Any]
34
35
39 -def get_pytorch_default_cudart_library_path() -> str:
40 - # code borrowed from https://github.com/pytorch/pytorch/blob/1cae60a87e5bdda8bcf55724a862eeed98a9747e/torch/__init__.py#L284 # noqa
41 - lib_folder = "cuda_runtime"
42 - lib_name = "libcudart.so.*[0-9]"
43 - lib_path = None
44 - for path in sys.path:
45 - nvidia_path = os.path.join(path, "nvidia")
46 - if not os.path.exists(nvidia_path):
47 - continue
48 - candidate_lib_paths = glob.glob(
49 - os.path.join(nvidia_path, lib_folder, "lib", lib_name))
50 - if candidate_lib_paths and not lib_path:
51 - lib_path = candidate_lib_paths[0]
52 - if lib_path:
53 - break
54 - if not lib_path:
55 - raise ValueError(f"{lib_name} not found in the system path {sys.path}")
56 - return lib_path
36 +def find_loaded_library(lib_name) -> Optional[str]:
37 + """
38 + According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,
39 + the file `/proc/self/maps` contains the memory maps of the process, which includes the
40 + shared libraries loaded by the process. We can use this file to find the path of the
41 + a loaded library.
42 + """ # noqa
43 + found = False
44 + with open("/proc/self/maps") as f:
45 + for line in f:
46 + if lib_name in line:
47 + found = True
48 + break
49 + if not found:
50 + # the library is not loaded in the current process
51 + return None
52 + start = line.index("/")
53 + path = line[start:].strip()
54 + return path
55
56
57 class CudaRTLibrary:
@@ -100,7 +98,9 @@ class CudaRTLibrary:
98
99 def __init__(self, so_file: Optional[str] = None):
100 if so_file is None:
103 - so_file = get_pytorch_default_cudart_library_path()
101 + so_file = find_loaded_library("libcudart.so")
102 + assert so_file is not None, \
103 + "libcudart.so is not loaded in the current process"
104 if so_file not in CudaRTLibrary.path_to_library_cache:
105 lib = ctypes.CDLL(so_file)
106 CudaRTLibrary.path_to_library_cache[so_file] = lib
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+3 -1
@@ -145,6 +145,7 @@ def can_actually_p2p(
145 p_tgt.start()
146 p_src.join()
147 p_tgt.join()
148 + assert p_src.exitcode == 0 and p_tgt.exitcode == 0
149 result: List[bool] = []
150 for src, tgt in zip(batch_src, batch_tgt):
151 a = result_queue.get()
@@ -221,7 +222,8 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
222 # wrap raised exception to provide more information
223 raise RuntimeError(
224 f"Error happened when batch testing "
224 - f"peer-to-peer access from {batch_src} to {batch_tgt}") from e
225 + f"peer-to-peer access from {batch_src} to {batch_tgt}:\n"
226 + f"{returned.stderr.decode()}") from e
227 result = pickle.loads(returned.stdout)
228 for _i, _j, r in zip(batch_src, batch_tgt, result):
229 cache[f"{_i}->{_j}"] = r