@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 042c3419fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Introduce VLLM_CUDART_SO_PATH to allow users specify the .so path (#12998)

Signed-off-by: Lu Fang <lufang@fb.com>

Lu Fang committed Feb 12, 2025 at 09:06 UTC 042c3419fad1a89c32a27abe8089af6de960bfce
2 files changed +37 -1
vllm/distributed/device_communicators/cuda_wrapper.py
+31 -1
@@ -5,12 +5,14 @@ convenient for use when we just need to call a few functions.
5 """
6
7 import ctypes
8 +import glob
9 from dataclasses import dataclass
10 from typing import Any, Dict, List, Optional
11
12 # this line makes it possible to directly load `libcudart.so` using `ctypes`
13 import torch # noqa
14
15 +import vllm.envs as envs
16 from vllm.logger import init_logger
17
18 logger = init_logger(__name__)
@@ -60,6 +62,29 @@ def find_loaded_library(lib_name) -> Optional[str]:
62 return path
63
64
65 +def get_cudart_lib_path_from_env() -> Optional[str]:
66 + """
67 + In some system, find_loaded_library() may not work. So we allow users to
68 + specify the path through environment variable VLLM_CUDART_SO_PATH.
69 + """
70 + cudart_so_env = envs.VLLM_CUDART_SO_PATH
71 + if cudart_so_env is not None:
72 + cudart_paths = [
73 + cudart_so_env,
74 + ]
75 + for path in cudart_paths:
76 + file_paths = glob.glob(path)
77 + if len(file_paths) > 0:
78 + logger.info(
79 + "Found cudart library at %s through env var"
80 + "VLLM_CUDART_SO_PATH=%s",
81 + file_paths[0],
82 + cudart_so_env,
83 + )
84 + return file_paths[0]
85 + return None
86 +
87 +
88 class CudaRTLibrary:
89 exported_functions = [
90 # ​cudaError_t cudaSetDevice ( int device )
@@ -105,8 +130,13 @@ class CudaRTLibrary:
130 def __init__(self, so_file: Optional[str] = None):
131 if so_file is None:
132 so_file = find_loaded_library("libcudart")
133 + if so_file is None:
134 + so_file = get_cudart_lib_path_from_env()
135 assert so_file is not None, \
109 - "libcudart is not loaded in the current process"
136 + (
137 + "libcudart is not loaded in the current process, "
138 + "try setting VLLM_CUDART_SO_PATH"
139 + )
140 if so_file not in CudaRTLibrary.path_to_library_cache:
141 lib = ctypes.CDLL(so_file)
142 CudaRTLibrary.path_to_library_cache[so_file] = lib
vllm/envs.py
+6
@@ -87,6 +87,7 @@ if TYPE_CHECKING:
87 VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False
88 VLLM_RAY_PER_WORKER_GPUS: float = 1.0
89 VLLM_RAY_BUNDLE_INDICES: str = ""
90 + VLLM_CUDART_SO_PATH: Optional[str] = None
91
92
93 def get_default_cache_root():
@@ -572,6 +573,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
573 # models the alignment is already naturally aligned to 256 bytes.
574 "VLLM_CUDA_MEM_ALIGN_KV_CACHE":
575 lambda: bool(int(os.getenv("VLLM_CUDA_MEM_ALIGN_KV_CACHE", "1"))),
576 +
577 + # In some system, find_loaded_library() may not work. So we allow users to
578 + # specify the path through environment variable VLLM_CUDART_SO_PATH.
579 + "VLLM_CUDART_SO_PATH":
580 + lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
581 }
582
583 # end-env-vars-definition