A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[distributed][misc] be consistent with pytorch for libcudart.so (#6346)
[distributed][misc] keep consistent with how pytorch finds libcudart.so (#6346)
youkaichao committed
Jul 11, 2024 at 19:35 UTC
2b0fb534813e9835077403723a484b7c03d47259
1 file changed
+24
-3
vllm/distributed/device_communicators/cuda_wrapper.py
+24
-3
@@ -4,6 +4,9 @@ convenient for use when we just need to call a few functions.
4
"""
5
6
import ctypes
7
+import glob
8
+import os
9
+import sys
10
from dataclasses import dataclass
11
from typing import Any, Dict, List, Optional
12
@@ -33,6 +36,26 @@ class Function:
36
argtypes: List[Any]
37
38
39
+def get_pytorch_default_cudart_library_path() -> str:
40
+ # code borrowed from https://github.com/pytorch/pytorch/blob/1cae60a87e5bdda8bcf55724a862eeed98a9747e/torch/__init__.py#L284 # noqa
41
+ lib_folder = "cuda_runtime"
42
+ lib_name = "libcudart.so.*[0-9]"
43
+ lib_path = None
44
+ for path in sys.path:
45
+ nvidia_path = os.path.join(path, "nvidia")
46
+ if not os.path.exists(nvidia_path):
47
+ continue
48
+ candidate_lib_paths = glob.glob(
49
+ os.path.join(nvidia_path, lib_folder, "lib", lib_name))
50
+ if candidate_lib_paths and not lib_path:
51
+ lib_path = candidate_lib_paths[0]
52
+ if lib_path:
53
+ break
54
+ if not lib_path:
55
+ raise ValueError(f"{lib_name} not found in the system path {sys.path}")
56
+ return lib_path
57
+
58
+
59
class CudaRTLibrary:
60
exported_functions = [
61
# cudaError_t cudaSetDevice ( int device )
@@ -77,9 +100,7 @@ class CudaRTLibrary:
100
101
def __init__(self, so_file: Optional[str] = None):
102
if so_file is None:
80
- assert torch.version.cuda is not None
81
- major_version = torch.version.cuda.split(".")[0]
82
- so_file = f"libcudart.so.{major_version}"
103
+ so_file = get_pytorch_default_cudart_library_path()
104
if so_file not in CudaRTLibrary.path_to_library_cache:
105
lib = ctypes.CDLL(so_file)
106
CudaRTLibrary.path_to_library_cache[so_file] = lib