A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[core][misc] update libcudart finding (#7620)
Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
youkaichao committed
Aug 16, 2024 at 23:01 UTC
d95cc0a55c55c914ad9fdb6a3a7ece282c334347
1 file changed
+7
-2
vllm/distributed/device_communicators/cuda_wrapper.py
+7
-2
@@ -49,8 +49,13 @@ def find_loaded_library(lib_name) -> Optional[str]:
49
if not found:
50
# the library is not loaded in the current process
51
return None
52
+ # if lib_name is libcudart, we need to match a line with:
53
+ # address /path/to/libcudart-hash.so.11.0
54
start = line.index("/")
55
path = line[start:].strip()
56
+ filename = path.split("/")[-1]
57
+ assert filename.rpartition(".so")[0].startswith(lib_name), \
58
+ f"Unexpected filename: {filename} for library {lib_name}"
59
return path
60
61
@@ -98,9 +103,9 @@ class CudaRTLibrary:
103
104
def __init__(self, so_file: Optional[str] = None):
105
if so_file is None:
101
- so_file = find_loaded_library("libcudart.so")
106
+ so_file = find_loaded_library("libcudart")
107
assert so_file is not None, \
103
- "libcudart.so is not loaded in the current process"
108
+ "libcudart is not loaded in the current process"
109
if so_file not in CudaRTLibrary.path_to_library_cache:
110
lib = ctypes.CDLL(so_file)
111
CudaRTLibrary.path_to_library_cache[so_file] = lib