@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d95cc0a55c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[core][misc] update libcudart finding (#7620)

Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>

youkaichao committed Aug 16, 2024 at 23:01 UTC d95cc0a55c55c914ad9fdb6a3a7ece282c334347
1 file changed +7 -2
vllm/distributed/device_communicators/cuda_wrapper.py
+7 -2
@@ -49,8 +49,13 @@ def find_loaded_library(lib_name) -> Optional[str]:
49 if not found:
50 # the library is not loaded in the current process
51 return None
52 + # if lib_name is libcudart, we need to match a line with:
53 + # address /path/to/libcudart-hash.so.11.0
54 start = line.index("/")
55 path = line[start:].strip()
56 + filename = path.split("/")[-1]
57 + assert filename.rpartition(".so")[0].startswith(lib_name), \
58 + f"Unexpected filename: {filename} for library {lib_name}"
59 return path
60
61
@@ -98,9 +103,9 @@ class CudaRTLibrary:
103
104 def __init__(self, so_file: Optional[str] = None):
105 if so_file is None:
101 - so_file = find_loaded_library("libcudart.so")
106 + so_file = find_loaded_library("libcudart")
107 assert so_file is not None, \
103 - "libcudart.so is not loaded in the current process"
108 + "libcudart is not loaded in the current process"
109 if so_file not in CudaRTLibrary.path_to_library_cache:
110 lib = ctypes.CDLL(so_file)
111 CudaRTLibrary.path_to_library_cache[so_file] = lib