@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2b0fb53481
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[distributed][misc] be consistent with pytorch for libcudart.so (#6346)

[distributed][misc] keep consistent with how pytorch finds libcudart.so (#6346)

youkaichao committed Jul 11, 2024 at 19:35 UTC 2b0fb534813e9835077403723a484b7c03d47259
1 file changed +24 -3
vllm/distributed/device_communicators/cuda_wrapper.py
+24 -3
@@ -4,6 +4,9 @@ convenient for use when we just need to call a few functions.
4 """
5
6 import ctypes
7 +import glob
8 +import os
9 +import sys
10 from dataclasses import dataclass
11 from typing import Any, Dict, List, Optional
12
@@ -33,6 +36,26 @@ class Function:
36 argtypes: List[Any]
37
38
39 +def get_pytorch_default_cudart_library_path() -> str:
40 + # code borrowed from https://github.com/pytorch/pytorch/blob/1cae60a87e5bdda8bcf55724a862eeed98a9747e/torch/__init__.py#L284 # noqa
41 + lib_folder = "cuda_runtime"
42 + lib_name = "libcudart.so.*[0-9]"
43 + lib_path = None
44 + for path in sys.path:
45 + nvidia_path = os.path.join(path, "nvidia")
46 + if not os.path.exists(nvidia_path):
47 + continue
48 + candidate_lib_paths = glob.glob(
49 + os.path.join(nvidia_path, lib_folder, "lib", lib_name))
50 + if candidate_lib_paths and not lib_path:
51 + lib_path = candidate_lib_paths[0]
52 + if lib_path:
53 + break
54 + if not lib_path:
55 + raise ValueError(f"{lib_name} not found in the system path {sys.path}")
56 + return lib_path
57 +
58 +
59 class CudaRTLibrary:
60 exported_functions = [
61 # ​cudaError_t cudaSetDevice ( int device )
@@ -77,9 +100,7 @@ class CudaRTLibrary:
100
101 def __init__(self, so_file: Optional[str] = None):
102 if so_file is None:
80 - assert torch.version.cuda is not None
81 - major_version = torch.version.cuda.split(".")[0]
82 - so_file = f"libcudart.so.{major_version}"
103 + so_file = get_pytorch_default_cudart_library_path()
104 if so_file not in CudaRTLibrary.path_to_library_cache:
105 lib = ctypes.CDLL(so_file)
106 CudaRTLibrary.path_to_library_cache[so_file] = lib