A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Introduce VLLM_CUDART_SO_PATH to allow users specify the .so path (#12998)
Signed-off-by: Lu Fang <lufang@fb.com>
Lu Fang committed
Feb 12, 2025 at 09:06 UTC
042c3419fad1a89c32a27abe8089af6de960bfce
2 files changed
+37
-1
vllm/distributed/device_communicators/cuda_wrapper.py
+31
-1
@@ -5,12 +5,14 @@ convenient for use when we just need to call a few functions.
5
"""
6
7
import ctypes
8
+import glob
9
from dataclasses import dataclass
10
from typing import Any, Dict, List, Optional
11
12
# this line makes it possible to directly load `libcudart.so` using `ctypes`
13
import torch # noqa
14
15
+import vllm.envs as envs
16
from vllm.logger import init_logger
17
18
logger = init_logger(__name__)
@@ -60,6 +62,29 @@ def find_loaded_library(lib_name) -> Optional[str]:
62
return path
63
64
65
+def get_cudart_lib_path_from_env() -> Optional[str]:
66
+ """
67
+ In some system, find_loaded_library() may not work. So we allow users to
68
+ specify the path through environment variable VLLM_CUDART_SO_PATH.
69
+ """
70
+ cudart_so_env = envs.VLLM_CUDART_SO_PATH
71
+ if cudart_so_env is not None:
72
+ cudart_paths = [
73
+ cudart_so_env,
74
+ ]
75
+ for path in cudart_paths:
76
+ file_paths = glob.glob(path)
77
+ if len(file_paths) > 0:
78
+ logger.info(
79
+ "Found cudart library at %s through env var"
80
+ "VLLM_CUDART_SO_PATH=%s",
81
+ file_paths[0],
82
+ cudart_so_env,
83
+ )
84
+ return file_paths[0]
85
+ return None
86
+
87
+
88
class CudaRTLibrary:
89
exported_functions = [
90
# cudaError_t cudaSetDevice ( int device )
@@ -105,8 +130,13 @@ class CudaRTLibrary:
130
def __init__(self, so_file: Optional[str] = None):
131
if so_file is None:
132
so_file = find_loaded_library("libcudart")
133
+ if so_file is None:
134
+ so_file = get_cudart_lib_path_from_env()
135
assert so_file is not None, \
109
- "libcudart is not loaded in the current process"
136
+ (
137
+ "libcudart is not loaded in the current process, "
138
+ "try setting VLLM_CUDART_SO_PATH"
139
+ )
140
if so_file not in CudaRTLibrary.path_to_library_cache:
141
lib = ctypes.CDLL(so_file)
142
CudaRTLibrary.path_to_library_cache[so_file] = lib
vllm/envs.py
+6
@@ -87,6 +87,7 @@ if TYPE_CHECKING:
87
VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False
88
VLLM_RAY_PER_WORKER_GPUS: float = 1.0
89
VLLM_RAY_BUNDLE_INDICES: str = ""
90
+ VLLM_CUDART_SO_PATH: Optional[str] = None
91
92
93
def get_default_cache_root():
@@ -572,6 +573,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
573
# models the alignment is already naturally aligned to 256 bytes.
574
"VLLM_CUDA_MEM_ALIGN_KV_CACHE":
575
lambda: bool(int(os.getenv("VLLM_CUDA_MEM_ALIGN_KV_CACHE", "1"))),
576
+
577
+ # In some system, find_loaded_library() may not work. So we allow users to
578
+ # specify the path through environment variable VLLM_CUDART_SO_PATH.
579
+ "VLLM_CUDART_SO_PATH":
580
+ lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
581
}
582
583
# end-env-vars-definition