A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Distributed] use absolute path for library file (#4271)
youkaichao committed
Apr 22, 2024 at 17:21 UTC
c1b4e4157c0b4154f950adaea85a259fc629c758
1 file changed
+30
-2
vllm/utils.py
+30
-2
@@ -553,6 +553,34 @@ def nccl_integrity_check(filepath):
553
return version.value
554
555
556
+@lru_cache(maxsize=None)
557
+def find_library(lib_name: str) -> str:
558
+ """
559
+ Find the library file in the system.
560
+ `lib_name` is full filename, with both prefix and suffix.
561
+ This function resolves `lib_name` to the full path of the library.
562
+ """
563
+ # Adapted from https://github.com/openai/triton/blob/main/third_party/nvidia/backend/driver.py#L19 # noqa
564
+ # According to https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
565
+ # `/sbin/ldconfig` should exist in all Linux systems.
566
+ # `/sbin/ldconfig` searches the library in the system
567
+ libs = subprocess.check_output(["/sbin/ldconfig", "-p"]).decode()
568
+ # each line looks like the following:
569
+ # libcuda.so.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libcuda.so.1
570
+ locs = [line.split()[-1] for line in libs.splitlines() if lib_name in line]
571
+ # `LD_LIBRARY_PATH` searches the library in the user-defined paths
572
+ env_ld_library_path = os.getenv("LD_LIBRARY_PATH")
573
+ if not locs and env_ld_library_path:
574
+ locs = [
575
+ os.path.join(dir, lib_name)
576
+ for dir in env_ld_library_path.split(":")
577
+ if os.path.exists(os.path.join(dir, lib_name))
578
+ ]
579
+ if not locs:
580
+ raise ValueError(f"Cannot find {lib_name} in the system.")
581
+ return locs[0]
582
+
583
+
584
def find_nccl_library():
585
so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
586
@@ -572,9 +600,9 @@ def find_nccl_library():
600
)
601
else:
602
if torch.version.cuda is not None:
575
- so_file = vllm_nccl_path or "libnccl.so.2"
603
+ so_file = vllm_nccl_path or find_library("libnccl.so.2")
604
elif torch.version.hip is not None:
577
- so_file = "librccl.so.1"
605
+ so_file = find_library("librccl.so.1")
606
else:
607
raise ValueError("NCCL only supports CUDA and ROCm backends.")
608
logger.info(f"Found nccl from library {so_file}")